Commit 29b52d398 for llama.cpp
commit 29b52d398b294509746ec0a3f28948a3107c0213
Author: Pedro Cuenca <pedro@huggingface.co>
Date: Fri Nov 20 23:01:07 2026 +0100
model : Inkling 2 Mini (#32694)
* Conversion
* Separate model_type
* Add gate_rank to allow building from metadata while testing
* Base model
* Loop graph
* Support the new is_varloop attribute
* Adopt naming scheme from https://github.com/ggml-org/llama.cpp/pull/32588
* Rename to varying
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 253a18fd4..8869b7252 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -148,7 +148,9 @@ TEXT_MODEL_MAP: dict[str, str] = {
"HYV4ForCausalLM": "hy_v4",
"IQuestCoderForCausalLM": "llama",
"Inkling2ForCausalLM": "inkling2",
+ "Inkling2LoopForCausalLM": "inkling2",
"Inkling2ForConditionalGeneration": "inkling2",
+ "Inkling2LoopForConditionalGeneration": "inkling2",
"InternLM2ForCausalLM": "internlm",
"InternLM3ForCausalLM": "internlm",
"JAISLMHeadModel": "jais",
@@ -389,6 +391,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"HunYuanVLForConditionalGeneration": "hunyuan",
"Idefics3ForConditionalGeneration": "smolvlm",
"Inkling2ForConditionalGeneration": "inkling2",
+ "Inkling2LoopForConditionalGeneration": "inkling2",
"InternVisionModel": "internvl",
"JanusForConditionalGeneration": "januspro",
"KimiK25ForConditionalGeneration": "kimivl",
diff --git a/conversion/inkling2.py b/conversion/inkling2.py
index a8e8b892e..63e73a6d5 100644
--- a/conversion/inkling2.py
+++ b/conversion/inkling2.py
@@ -69,7 +69,7 @@ def _is_inkling2(dir_model: Path) -> bool:
if not (dir_model / "config.json").is_file():
return False
with open(dir_model / "config.json", encoding="utf-8") as f:
- return json.load(f).get("model_type") in ("inkling2", "inkling2_text")
+ return json.load(f).get("model_type") in ("inkling2", "inkling2_text", "inkling2_loop", "inkling2_loop_text")
@ModelBase.register_hparams_loader(_is_inkling2)
@@ -274,7 +274,62 @@ class Inkling2Model(TextModel):
yield from super().modify_tensors(data_torch, name, bid)
-@ModelBase.register("Inkling2ForConditionalGeneration")
+@ModelBase.register("Inkling2LoopForConditionalGeneration", "Inkling2LoopForCausalLM")
+class Inkling2LoopModel(Inkling2Model):
+ model_arch = gguf.MODEL_ARCH.INKLING2_LOOP
+
+ # raw checkpoint and transformers names of the cross-pass message queries
+ _query_kinds = {
+ "model.middle_cycle_cross_pass_queries": "q",
+ "model.middle_cycle_cross_pass_kv_queries": "kv",
+ "model.cross_pass_messages.queries": "q",
+ "model.cross_pass_messages.kv_queries": "kv",
+ }
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self._queries: dict[str, Tensor] = {}
+ self.loop = self.hparams["loop_config"]
+ if set(self.loop) != {"core_start", "core_end", "num_passes", "gate_rank", "is_varloop"}:
+ raise ValueError(f"Inkling2Loop: unsupported loop_config keys {sorted(self.loop)}")
+ start, end, block = self.loop["core_start"], self.loop["core_end"], self.hparams["n_layers_per_attn_res_block"]
+ # a fixed loop needs at least 2 passes, a variable loop at least 1
+ min_passes = 1 if self.loop["is_varloop"] else 2
+ if not 0 < start < end <= self.block_count or start % block or end % block or self.loop["num_passes"] < min_passes or self.loop["gate_rank"] < 1:
+ raise ValueError(f"Inkling2Loop: invalid loop_config {self.loop}")
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_loop_start(self.loop["core_start"])
+ self.gguf_writer.add_loop_end(self.loop["core_end"])
+ self.gguf_writer.add_loop_count(self.loop["num_passes"])
+ self.gguf_writer.add_loop_gate_rank(self.loop["gate_rank"])
+ self.gguf_writer.add_loop_varying(self.loop["is_varloop"])
+
+ def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
+ # the reference does not quantize the loop tensors
+ if new_name.endswith(("loop_msg_query.weight", "_ctrl_down.weight", "_ctrl_gain.weight", "_ctrl_forget.weight")):
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name not in self._query_kinds:
+ yield from super().modify_tensors(data_torch, name, bid)
+ return
+ self._queries[self._query_kinds[name]] = data_torch
+ if len(self._queries) < 2:
+ return
+ q, kv = self._queries["q"], self._queries["kv"]
+ start = self.loop["core_start"]
+ if q.shape[0] != 2 * (self.loop["core_end"] - start) or kv.shape != q.shape:
+ raise ValueError(f"Inkling2Loop: cross-pass queries {tuple(q.shape)} and {tuple(kv.shape)} do not match the loop core")
+ # core layer start + i mixes its q, k, v and mlp messages with these four queries
+ for i in range(q.shape[0] // 2):
+ query = torch.stack([q[2 * i], kv[2 * i], kv[2 * i + 1], q[2 * i + 1]])
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.LOOP_MSG_QUERY, start + i, ".weight"), query
+
+
+@ModelBase.register("Inkling2ForConditionalGeneration", "Inkling2LoopForConditionalGeneration")
class Inkling2VisionModel(MmprojModel):
def get_vision_config(self):
config = self.global_config.get("vision_config")
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index f36566a81..cc0347bba 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -326,6 +326,13 @@ class Keys:
L_CYCLES = "{arch}.hrm.l_cycles"
PREFIX_LM = "{arch}.hrm.prefix_lm"
+ class Loop:
+ START = "{arch}.loop.start"
+ END = "{arch}.loop.end"
+ COUNT = "{arch}.loop.count"
+ GATE_RANK = "{arch}.loop.gate_rank"
+ VARYING = "{arch}.loop.varying"
+
class Metis:
PASS_COUNT = "{arch}.metis.pass_count"
MEMORY_LENGTH = "{arch}.metis.memory_length"
@@ -778,6 +785,7 @@ class MODEL_ARCH(IntEnum):
KOLIBRI1 = auto()
LIMITE = auto()
INKLING2 = auto()
+ INKLING2_LOOP = auto()
DOMYN_EDGE = auto()
@@ -880,6 +888,19 @@ class MODEL_TENSOR(IntEnum):
MOE_LATENT_UP = auto() # nemotron 3 super
FFN_POST_CONV = auto()
FFN_RES_QUERY = auto()
+ LOOP_MSG_QUERY = auto()
+ LOOP_Q_CTRL_DOWN = auto()
+ LOOP_Q_CTRL_GAIN = auto()
+ LOOP_Q_CTRL_FORGET = auto()
+ LOOP_K_CTRL_DOWN = auto()
+ LOOP_K_CTRL_GAIN = auto()
+ LOOP_K_CTRL_FORGET = auto()
+ LOOP_V_CTRL_DOWN = auto()
+ LOOP_V_CTRL_GAIN = auto()
+ LOOP_V_CTRL_FORGET = auto()
+ LOOP_FFN_CTRL_DOWN = auto()
+ LOOP_FFN_CTRL_GAIN = auto()
+ LOOP_FFN_CTRL_FORGET = auto()
STREAM_EMBD = auto()
DEPTH_MEM_STATE_WRITE = auto()
DEPTH_MEM_META_WRITE = auto()
@@ -1779,6 +1800,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.KOLIBRI1: "kolibri1",
MODEL_ARCH.LIMITE: "limite",
MODEL_ARCH.INKLING2: "inkling2",
+ MODEL_ARCH.INKLING2_LOOP: "inkling2_loop",
MODEL_ARCH.DOMYN_EDGE: "domynedge",
}
@@ -1881,6 +1903,19 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super
MODEL_TENSOR.FFN_POST_CONV: "blk.{bid}.ffn_post_conv",
MODEL_TENSOR.FFN_RES_QUERY: "blk.{bid}.ffn_res_query",
+ MODEL_TENSOR.LOOP_MSG_QUERY: "blk.{bid}.loop_msg_query",
+ MODEL_TENSOR.LOOP_Q_CTRL_DOWN: "blk.{bid}.loop_q_ctrl_down",
+ MODEL_TENSOR.LOOP_Q_CTRL_GAIN: "blk.{bid}.loop_q_ctrl_gain",
+ MODEL_TENSOR.LOOP_Q_CTRL_FORGET: "blk.{bid}.loop_q_ctrl_forget",
+ MODEL_TENSOR.LOOP_K_CTRL_DOWN: "blk.{bid}.loop_k_ctrl_down",
+ MODEL_TENSOR.LOOP_K_CTRL_GAIN: "blk.{bid}.loop_k_ctrl_gain",
+ MODEL_TENSOR.LOOP_K_CTRL_FORGET: "blk.{bid}.loop_k_ctrl_forget",
+ MODEL_TENSOR.LOOP_V_CTRL_DOWN: "blk.{bid}.loop_v_ctrl_down",
+ MODEL_TENSOR.LOOP_V_CTRL_GAIN: "blk.{bid}.loop_v_ctrl_gain",
+ MODEL_TENSOR.LOOP_V_CTRL_FORGET: "blk.{bid}.loop_v_ctrl_forget",
+ MODEL_TENSOR.LOOP_FFN_CTRL_DOWN: "blk.{bid}.loop_ffn_ctrl_down",
+ MODEL_TENSOR.LOOP_FFN_CTRL_GAIN: "blk.{bid}.loop_ffn_ctrl_gain",
+ MODEL_TENSOR.LOOP_FFN_CTRL_FORGET: "blk.{bid}.loop_ffn_ctrl_forget",
MODEL_TENSOR.STREAM_EMBD: "stream_embd",
MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: "depth_mem_state_write",
MODEL_TENSOR.DEPTH_MEM_META_WRITE: "depth_mem_meta_write",
@@ -7087,6 +7122,55 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_POST_CONV,
MODEL_TENSOR.FFN_RES_QUERY,
],
+ MODEL_ARCH.INKLING2_LOOP: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT_RES_QUERY,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_REL,
+ MODEL_TENSOR.ATTN_REL_BIAS,
+ MODEL_TENSOR.ATTN_Q_CLIP,
+ MODEL_TENSOR.ATTN_REL_CLIP,
+ MODEL_TENSOR.ATTN_SINK_PROJ,
+ MODEL_TENSOR.ATTN_Q_INP_NORM,
+ MODEL_TENSOR.ATTN_K_INP_NORM,
+ MODEL_TENSOR.ATTN_V_INP_NORM,
+ MODEL_TENSOR.ATTN_K_CONV,
+ MODEL_TENSOR.ATTN_V_CONV,
+ MODEL_TENSOR.ATTN_K_CONV_PROJ,
+ MODEL_TENSOR.ATTN_V_CONV_PROJ,
+ MODEL_TENSOR.ATTN_POST_CONV,
+ MODEL_TENSOR.ATTN_RES_QUERY,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_UP_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_POST_CONV,
+ MODEL_TENSOR.FFN_RES_QUERY,
+ MODEL_TENSOR.LOOP_MSG_QUERY,
+ MODEL_TENSOR.LOOP_Q_CTRL_DOWN,
+ MODEL_TENSOR.LOOP_Q_CTRL_GAIN,
+ MODEL_TENSOR.LOOP_Q_CTRL_FORGET,
+ MODEL_TENSOR.LOOP_K_CTRL_DOWN,
+ MODEL_TENSOR.LOOP_K_CTRL_GAIN,
+ MODEL_TENSOR.LOOP_K_CTRL_FORGET,
+ MODEL_TENSOR.LOOP_V_CTRL_DOWN,
+ MODEL_TENSOR.LOOP_V_CTRL_GAIN,
+ MODEL_TENSOR.LOOP_V_CTRL_FORGET,
+ MODEL_TENSOR.LOOP_FFN_CTRL_DOWN,
+ MODEL_TENSOR.LOOP_FFN_CTRL_GAIN,
+ MODEL_TENSOR.LOOP_FFN_CTRL_FORGET,
+ ],
MODEL_ARCH.DOMYN_EDGE: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.ATTN_Q,
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index eb0bd91d0..08c8d2f33 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -990,6 +990,21 @@ class GGUFWriter:
def add_hrm_prefix_lm(self, value: bool) -> None:
self.add_bool(Keys.HRM.PREFIX_LM.format(arch=self.arch), value)
+ def add_loop_start(self, value: int) -> None:
+ self.add_uint32(Keys.Loop.START.format(arch=self.arch), value)
+
+ def add_loop_end(self, value: int) -> None:
+ self.add_uint32(Keys.Loop.END.format(arch=self.arch), value)
+
+ def add_loop_count(self, value: int) -> None:
+ self.add_uint32(Keys.Loop.COUNT.format(arch=self.arch), value)
+
+ def add_loop_gate_rank(self, value: int) -> None:
+ self.add_uint32(Keys.Loop.GATE_RANK.format(arch=self.arch), value)
+
+ def add_loop_varying(self, value: bool) -> None:
+ self.add_bool(Keys.Loop.VARYING.format(arch=self.arch), value)
+
def add_metis_pass_count(self, value: int) -> None:
self.add_uint32(Keys.Metis.PASS_COUNT.format(arch=self.arch), value)
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index d48d46b9a..c0bbfde22 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -543,6 +543,66 @@ class TensorNameMap:
"model.layers.{bid}.attention_input.query", # inkling2
),
+ MODEL_TENSOR.LOOP_Q_CTRL_DOWN: (
+ "model.layers.{bid}.q_controller.down_proj", # inkling2_loop
+ "model.layers.{bid}.controllers.attn.down_weight", # inkling2_loop
+ ),
+
+ MODEL_TENSOR.LOOP_Q_CTRL_GAIN: (
+ "model.layers.{bid}.q_controller.gain_proj", # inkling2_loop
+ "model.layers.{bid}.controllers.attn.gain_weight", # inkling2_loop
+ ),
+
+ MODEL_TENSOR.LOOP_Q_CTRL_FORGET: (
+ "model.layers.{bid}.q_controller.forget_proj", # inkling2_loop
+ "model.layers.{bid}.controllers.attn.forget_weight", # inkling2_loop
+ ),
+
+ MODEL_TENSOR.LOOP_K_CTRL_DOWN: (
+ "model.layers.{bid}.k_controller.down_proj", # inkling2_loop
+ "model.layers.{bid}.controllers.k.down_weight", # inkling2_loop
+ ),
+
+ MODEL_TENSOR.LOOP_K_CTRL_GAIN: (
+ "model.layers.{bid}.k_controller.gain_proj", # inkling2_loop
+ "model.layers.{bid}.controllers.k.gain_weight", # inkling2_loop
+ ),
+
+ MODEL_TENSOR.LOOP_K_CTRL_FORGET: (
+ "model.layers.{bid}.k_controller.forget_proj", # inkling2_loop
+ "model.layers.{bid}.controllers.k.forget_weight", # inkling2_loop
+ ),
+
+ MODEL_TENSOR.LOOP_V_CTRL_DOWN: (
+ "model.layers.{bid}.v_controller.down_proj", # inkling2_loop
+ "model.layers.{bid}.controllers.v.down_weight", # inkling2_loop
+ ),
+
+ MODEL_TENSOR.LOOP_V_CTRL_GAIN: (
+ "model.layers.{bid}.v_controller.gain_proj", # inkling2_loop
+ "model.layers.{bid}.controllers.v.gain_weight", # inkling2_loop
+ ),
+
+ MODEL_TENSOR.LOOP_V_CTRL_FORGET: (
+ "model.layers.{bid}.v_controller.forget_proj", # inkling2_loop
+ "model.layers.{bid}.controllers.v.forget_weight", # inkling2_loop
+ ),
+
+ MODEL_TENSOR.LOOP_FFN_CTRL_DOWN: (
+ "model.layers.{bid}.mlp_controller.down_proj", # inkling2_loop
+ "model.layers.{bid}.controllers.mlp.down_weight", # inkling2_loop
+ ),
+
+ MODEL_TENSOR.LOOP_FFN_CTRL_GAIN: (
+ "model.layers.{bid}.mlp_controller.gain_proj", # inkling2_loop
+ "model.layers.{bid}.controllers.mlp.gain_weight", # inkling2_loop
+ ),
+
+ MODEL_TENSOR.LOOP_FFN_CTRL_FORGET: (
+ "model.layers.{bid}.mlp_controller.forget_proj", # inkling2_loop
+ "model.layers.{bid}.controllers.mlp.forget_weight", # inkling2_loop
+ ),
+
# Feed-forward norm
MODEL_TENSOR.FFN_NORM: (
"layers.{bid}.moe_connection.norm", # metis16
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 1c90b2180..3cda4f7fb 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -184,6 +184,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_KOLIBRI1, "kolibri1" },
{ LLM_ARCH_RICURSIVE, "ricursive" },
{ LLM_ARCH_INKLING2, "inkling2" },
+ { LLM_ARCH_INKLING2_LOOP, "inkling2_loop" },
{ LLM_ARCH_DOMYN_EDGE, "domynedge" },
{ LLM_ARCH_UNKNOWN, "(unknown)" },
};
@@ -285,6 +286,11 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_HRM_H_CYCLES, "%s.hrm.h_cycles" },
{ LLM_KV_HRM_L_CYCLES, "%s.hrm.l_cycles" },
{ LLM_KV_HRM_PREFIX_LM, "%s.hrm.prefix_lm" },
+ { LLM_KV_LOOP_START, "%s.loop.start" },
+ { LLM_KV_LOOP_END, "%s.loop.end" },
+ { LLM_KV_LOOP_COUNT, "%s.loop.count" },
+ { LLM_KV_LOOP_GATE_RANK, "%s.loop.gate_rank" },
+ { LLM_KV_LOOP_VARYING, "%s.loop.varying" },
{ LLM_KV_METIS_PASS_COUNT, "%s.metis.pass_count" },
{ LLM_KV_METIS_MEMORY_LENGTH, "%s.metis.memory_length" },
{ LLM_KV_METIS_ROUTE_LENGTH, "%s.metis.route_length" },
@@ -591,6 +597,19 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_FFN_LATENT_UP, "blk.%d.ffn_latent_up" },
{ LLM_TENSOR_FFN_POST_CONV, "blk.%d.ffn_post_conv" },
{ LLM_TENSOR_FFN_RES_QUERY, "blk.%d.ffn_res_query" },
+ { LLM_TENSOR_LOOP_MSG_QUERY, "blk.%d.loop_msg_query" },
+ { LLM_TENSOR_LOOP_Q_CTRL_DOWN, "blk.%d.loop_q_ctrl_down" },
+ { LLM_TENSOR_LOOP_Q_CTRL_GAIN, "blk.%d.loop_q_ctrl_gain" },
+ { LLM_TENSOR_LOOP_Q_CTRL_FORGET, "blk.%d.loop_q_ctrl_forget" },
+ { LLM_TENSOR_LOOP_K_CTRL_DOWN, "blk.%d.loop_k_ctrl_down" },
+ { LLM_TENSOR_LOOP_K_CTRL_GAIN, "blk.%d.loop_k_ctrl_gain" },
+ { LLM_TENSOR_LOOP_K_CTRL_FORGET, "blk.%d.loop_k_ctrl_forget" },
+ { LLM_TENSOR_LOOP_V_CTRL_DOWN, "blk.%d.loop_v_ctrl_down" },
+ { LLM_TENSOR_LOOP_V_CTRL_GAIN, "blk.%d.loop_v_ctrl_gain" },
+ { LLM_TENSOR_LOOP_V_CTRL_FORGET, "blk.%d.loop_v_ctrl_forget" },
+ { LLM_TENSOR_LOOP_FFN_CTRL_DOWN, "blk.%d.loop_ffn_ctrl_down" },
+ { LLM_TENSOR_LOOP_FFN_CTRL_GAIN, "blk.%d.loop_ffn_ctrl_gain" },
+ { LLM_TENSOR_LOOP_FFN_CTRL_FORGET, "blk.%d.loop_ffn_ctrl_forget" },
{ LLM_TENSOR_STREAM_EMBD, "stream_embd" },
{ LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "depth_mem_state_write" },
{ LLM_TENSOR_DEPTH_MEM_META_WRITE, "depth_mem_meta_write" },
@@ -1262,6 +1281,19 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_FFN_LATENT_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_FFN_POST_CONV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_SSM_CONV}},
{LLM_TENSOR_FFN_RES_QUERY, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_MSG_QUERY, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_Q_CTRL_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_Q_CTRL_GAIN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_Q_CTRL_FORGET, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_K_CTRL_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_K_CTRL_GAIN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_K_CTRL_FORGET, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_V_CTRL_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_V_CTRL_GAIN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_V_CTRL_FORGET, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_FFN_CTRL_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_FFN_CTRL_GAIN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_LOOP_FFN_CTRL_FORGET, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_STREAM_EMBD, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_ADD}},
{LLM_TENSOR_DEPTH_MEM_STATE_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DEPTH_MEM_META_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
@@ -1448,6 +1480,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
case LLM_ARCH_MINIMAX_01:
case LLM_ARCH_ZONOS3:
case LLM_ARCH_INKLING2:
+ case LLM_ARCH_INKLING2_LOOP:
return true;
default:
return false;
@@ -1484,6 +1517,7 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
case LLM_ARCH_LFM3:
case LLM_ARCH_BAILINGMOE3:
case LLM_ARCH_INKLING2:
+ case LLM_ARCH_INKLING2_LOOP:
return true;
default:
return false;
@@ -1502,6 +1536,7 @@ bool llm_arch_supports_mixed_batch(const llm_arch & arch) {
case LLM_ARCH_GEMMA4_ASSISTANT:
case LLM_ARCH_LIMITE:
case LLM_ARCH_INKLING2:
+ case LLM_ARCH_INKLING2_LOOP:
return false;
default:
return true;
@@ -1522,6 +1557,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_JAMBA:
case LLM_ARCH_FALCON_H1:
case LLM_ARCH_INKLING2:
+ case LLM_ARCH_INKLING2_LOOP:
case LLM_ARCH_OLMO2:
case LLM_ARCH_OLMOE:
case LLM_ARCH_DEEPSEEK2:
@@ -1564,6 +1600,7 @@ bool llama_arch_supports_save(llm_arch arch) {
case LLM_ARCH_APERTUS:
case LLM_ARCH_STEP35:
case LLM_ARCH_CLEF: // the head tensors are not saved
+ case LLM_ARCH_INKLING2_LOOP: // the loop tensors are not saved and the cache slots are expanded at load
case LLM_ARCH_VLLM_DECISION2:
return false;
default:
diff --git a/src/llama-arch.h b/src/llama-arch.h
index a8c0fdf09..1ac22ecee 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -188,6 +188,7 @@ enum llm_arch {
LLM_ARCH_LIMITE,
LLM_ARCH_RICURSIVE,
LLM_ARCH_INKLING2,
+ LLM_ARCH_INKLING2_LOOP,
LLM_ARCH_DOMYN_EDGE,
LLM_ARCH_UNKNOWN,
};
@@ -289,6 +290,11 @@ enum llm_kv {
LLM_KV_HRM_H_CYCLES,
LLM_KV_HRM_L_CYCLES,
LLM_KV_HRM_PREFIX_LM,
+ LLM_KV_LOOP_START,
+ LLM_KV_LOOP_END,
+ LLM_KV_LOOP_COUNT,
+ LLM_KV_LOOP_GATE_RANK,
+ LLM_KV_LOOP_VARYING,
LLM_KV_METIS_PASS_COUNT,
LLM_KV_METIS_MEMORY_LENGTH,
LLM_KV_METIS_ROUTE_LENGTH,
@@ -610,6 +616,19 @@ enum llm_tensor {
LLM_TENSOR_FFN_LATENT_UP,
LLM_TENSOR_FFN_POST_CONV,
LLM_TENSOR_FFN_RES_QUERY,
+ LLM_TENSOR_LOOP_MSG_QUERY,
+ LLM_TENSOR_LOOP_Q_CTRL_DOWN,
+ LLM_TENSOR_LOOP_Q_CTRL_GAIN,
+ LLM_TENSOR_LOOP_Q_CTRL_FORGET,
+ LLM_TENSOR_LOOP_K_CTRL_DOWN,
+ LLM_TENSOR_LOOP_K_CTRL_GAIN,
+ LLM_TENSOR_LOOP_K_CTRL_FORGET,
+ LLM_TENSOR_LOOP_V_CTRL_DOWN,
+ LLM_TENSOR_LOOP_V_CTRL_GAIN,
+ LLM_TENSOR_LOOP_V_CTRL_FORGET,
+ LLM_TENSOR_LOOP_FFN_CTRL_DOWN,
+ LLM_TENSOR_LOOP_FFN_CTRL_GAIN,
+ LLM_TENSOR_LOOP_FFN_CTRL_FORGET,
LLM_TENSOR_STREAM_EMBD,
LLM_TENSOR_DEPTH_MEM_STATE_WRITE,
LLM_TENSOR_DEPTH_MEM_META_WRITE,
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index ec0820a07..4c77f6f68 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -326,6 +326,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_falcon_h1(params);
case LLM_ARCH_INKLING2:
return new llama_model_inkling2(params);
+ case LLM_ARCH_INKLING2_LOOP:
+ return new llama_model_inkling2_loop(params);
case LLM_ARCH_LFM2:
return new llama_model_lfm2(params);
case LLM_ARCH_LFM2MOE:
@@ -3359,7 +3361,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
filter_idx = [&](uint32_t il) {
return il < hparams.n_layer() && !hparams.is_recr(il) && hparams.is_indexer_full(il);
};
- } else if (arch == LLM_ARCH_FALCON_H1 || arch == LLM_ARCH_INKLING2 || arch == LLM_ARCH_ZAYA1) {
+ } else if (arch == LLM_ARCH_FALCON_H1 || arch == LLM_ARCH_INKLING2 || arch == LLM_ARCH_INKLING2_LOOP || arch == LLM_ARCH_ZAYA1) {
filter_attn = [&](uint32_t) { return true; };
filter_recr = [&](uint32_t) { return true; };
} else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
@@ -3735,6 +3737,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_GLM5_NEXT:
case LLM_ARCH_STARCODER:
case LLM_ARCH_INKLING2:
+ case LLM_ARCH_INKLING2_LOOP:
case LLM_ARCH_KOKORO:
return LLAMA_ROPE_TYPE_NONE;
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index 79d964972..0515756dc 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -340,12 +340,13 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
quantize &= name.find("mudd_") == std::string::npos;
}
- // inkling2: short conv kernels, clip scales, residual mix queries and the relative bias projection
- if (arch == LLM_ARCH_INKLING2) {
+ // inkling2: short conv kernels, clip scales, residual mix queries, the relative bias projection and the loop tensors
+ if (arch == LLM_ARCH_INKLING2 || arch == LLM_ARCH_INKLING2_LOOP) {
quantize &= name.find("_conv.weight") == std::string::npos;
quantize &= name.find("_clip.weight") == std::string::npos;
quantize &= name.find("_res_query.weight") == std::string::npos;
quantize &= name.find("attn_rel_bias.weight") == std::string::npos;
+ quantize &= name.find(".loop_") == std::string::npos;
}
// zaya1: the top-1 router and the grouped conv kernels stay F32
diff --git a/src/models/inkling2-loop.cpp b/src/models/inkling2-loop.cpp
new file mode 100644
index 000000000..0de66baee
--- /dev/null
+++ b/src/models/inkling2-loop.cpp
@@ -0,0 +1,113 @@
+#include "models.h"
+
+int llama_model_inkling2_loop::layer_phys(int il) const {
+ return il < n_phys ? il : core_start + (il - n_phys) % (core_end - core_start);
+}
+
+void llama_model_inkling2_loop::load_arch_hparams(llama_model_loader & ml) {
+ llama_model_inkling2::load_arch_hparams(ml);
+
+ ml.get_key(LLM_KV_LOOP_START, core_start);
+ ml.get_key(LLM_KV_LOOP_END, core_end);
+ ml.get_key(LLM_KV_LOOP_COUNT, n_passes);
+ ml.get_key(LLM_KV_LOOP_GATE_RANK, gate_rank);
+ ml.get_key(LLM_KV_LOOP_VARYING, varying, false);
+
+ n_phys = hparams.n_layer();
+ const uint32_t block = hparams.attn_res_block_size;
+ // a fixed loop needs at least 2 passes, a variable loop at least 1
+ if (n_passes < (varying ? 1u : 2u) || !gate_rank || !core_start || core_start >= core_end || core_end > (uint32_t) n_phys ||
+ core_start % block || core_end % block || hparams.n_layer_nextn) {
+ throw std::runtime_error(format("inkling2_loop: invalid loop core [%u, %u) with block size %u, %u passes or gate rank %u for %d layers",
+ core_start, core_end, block, n_passes, gate_rank, n_phys));
+ }
+ const size_t n_slot = n_phys + (size_t) (n_passes - 1)*(core_end - core_start);
+ if (n_slot > LLAMA_MAX_LAYERS) {
+ throw std::runtime_error(format("inkling2_loop: %zu cache slots exceed the maximum of %d", n_slot, LLAMA_MAX_LAYERS));
+ }
+
+ // expand to the cache slots before load_tensors() allocates the layers and the memory
+ hparams.n_layer_all = n_slot;
+ for (int il = n_phys; il < (int) n_slot; ++il) {
+ const int p = layer_phys(il);
+ hparams.n_head_arr[il] = hparams.n_head_arr[p];
+ hparams.n_head_kv_arr[il] = hparams.n_head_kv_arr[p];
+ hparams.n_ff_arr[il] = hparams.n_ff_arr[p];
+ hparams.n_ff_exp_arr[il] = hparams.n_ff_exp_arr[p];
+ hparams.n_expert_used_arr[il] = hparams.n_expert_used_arr[p];
+ hparams.is_swa_impl[il] = hparams.is_swa_impl[p];
+ hparams.is_recr_impl[il] = hparams.is_recr_impl[p];
+ }
+}
+
+void llama_model_inkling2_loop::load_arch_tensors(llama_model_loader & ml) {
+ llama_model_inkling2::load_arch_tensors(ml);
+
+ const int64_t n_embd = hparams.n_embd;
+ const int64_t rank = gate_rank;
+ const llm_tensor down[LOOP_CTRL_COUNT] = { LLM_TENSOR_LOOP_Q_CTRL_DOWN, LLM_TENSOR_LOOP_K_CTRL_DOWN, LLM_TENSOR_LOOP_V_CTRL_DOWN, LLM_TENSOR_LOOP_FFN_CTRL_DOWN };
+ const llm_tensor gain[LOOP_CTRL_COUNT] = { LLM_TENSOR_LOOP_Q_CTRL_GAIN, LLM_TENSOR_LOOP_K_CTRL_GAIN, LLM_TENSOR_LOOP_V_CTRL_GAIN, LLM_TENSOR_LOOP_FFN_CTRL_GAIN };
+ const llm_tensor forget[LOOP_CTRL_COUNT] = { LLM_TENSOR_LOOP_Q_CTRL_FORGET, LLM_TENSOR_LOOP_K_CTRL_FORGET, LLM_TENSOR_LOOP_V_CTRL_FORGET, LLM_TENSOR_LOOP_FFN_CTRL_FORGET };
+
+ loop_layers.resize(core_end - core_start);
+ for (uint32_t il = core_start; il < core_end; ++il) {
+ auto & l = loop_layers[il - core_start];
+ // the q, k, v and ffn message queries
+ l.msg_query = create_tensor(tn(LLM_TENSOR_LOOP_MSG_QUERY, "weight", il), {n_embd, LOOP_CTRL_COUNT}, 0);
+ for (int c = 0; c < LOOP_CTRL_COUNT; ++c) {
+ l.ctrl_down[c] = create_tensor(tn(down[c], "weight", il), {2*n_embd, rank}, 0);
+ l.ctrl_gain[c] = create_tensor(tn(gain[c], "weight", il), {rank, n_embd}, 0);
+ l.ctrl_forget[c] = create_tensor(tn(forget[c], "weight", il), {rank, n_embd}, 0);
+ }
+ }
+
+ // share the physical weights, each cache slot keeps its own KV and conv state
+ for (int il = n_phys; il < (int) hparams.n_layer(); ++il) {
+ layers[il] = layers[layer_phys(il)];
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_inkling2_loop::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(params);
+}
+
+llama_model_inkling2_loop::graph::graph(const llm_graph_params & params) : llama_model_inkling2::graph(params, no_build{}) {
+ const auto & m = static_cast<const llama_model_inkling2_loop &>(model);
+ const int start = m.core_start;
+ const int end = m.core_end;
+
+ auto * cur = build_inputs();
+ std::vector<ggml_tensor *> blocks;
+ for (int il = 0; il < start; ++il) {
+ cur = build_layer(il, il, cur, blocks);
+ }
+
+ // every pass starts from the prelude output
+ auto * prelude = cur;
+ const auto prelude_blocks = blocks;
+ for (uint32_t pass = 0; pass < m.n_passes; ++pass) {
+ // the bank of the previous pass: its blocks after the first core block, and its output
+ std::vector<ggml_tensor *> bank;
+ ggml_tensor * bank_out = nullptr;
+ if (pass > 0) {
+ bank.assign(blocks.begin() + prelude_blocks.size() + 1, blocks.end());
+ bank_out = cur;
+ cur = prelude;
+ blocks = prelude_blocks;
+ }
+ for (int ip = start; ip < end; ++ip) {
+ const int il = pass == 0 ? ip : m.n_phys + (pass - 1)*(end - start) + (ip - start);
+ if (pass == 0) {
+ cur = build_layer(il, ip, cur, blocks);
+ continue;
+ }
+ const auto & loop = m.loop_layers[ip - start];
+ cur = build_layer(il, ip, cur, blocks, mix(bank, bank_out, loop.msg_query), &loop);
+ }
+ }
+
+ for (int il = end; il < m.n_phys; ++il) {
+ cur = build_layer(il, il, cur, blocks);
+ }
+ build_output(cur, blocks);
+}
diff --git a/src/models/inkling2.cpp b/src/models/inkling2.cpp
index ca25c1d53..3e43ef418 100644
--- a/src/models/inkling2.cpp
+++ b/src/models/inkling2.cpp
@@ -100,7 +100,7 @@ void llama_model_inkling2::load_arch_tensors(llama_model_loader &) {
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
output_res_query = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_QUERY, "weight"), {n_embd}, 0);
- for (int i = 0; i < n_layer; ++i) {
+ for (int i = 0; i < n_layer_phys(); ++i) {
auto & layer = layers[i];
const int64_t n_head = hparams.n_head(i);
@@ -236,16 +236,27 @@ ggml_tensor * llama_model_inkling2::graph::convolve(
return ggml_scale(ctx0, ggml_add(ctx0, cur, out), 1.0f/sqrtf((dynamic ? 2 : 1)*n_conv + 1));
}
-llama_model_inkling2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) {
- auto * inp = build_inp_mem_hybrid_iswa_rel(hparams.n_relative_extent, hparams.n_swa);
- auto * attn = inp->get_attn();
- auto * recr = inp->get_recr();
+// mix a cross-pass message into a looped layer input: (1 - forget)*cur + gain*message, both gated on cur and message
+ggml_tensor * llama_model_inkling2::graph::control(ggml_tensor * cur, ggml_tensor * message, const loop_layer & loop, int ctrl) {
+ ggml_tensor * gate = ggml_concat(ctx0, ggml_rms_norm(ctx0, cur, 1e-6f), ggml_rms_norm(ctx0, message, 1e-6f), 0);
+ gate = build_lora_mm(loop.ctrl_down[ctrl], gate);
+
+ ggml_tensor * gain = ggml_tanh(ctx0, build_lora_mm(loop.ctrl_gain[ctrl], gate));
+ ggml_tensor * forget = ggml_tanh(ctx0, build_lora_mm(loop.ctrl_forget[ctrl], gate));
+
+ return ggml_add(ctx0, ggml_sub(ctx0, cur, ggml_mul(ctx0, forget, cur)), ggml_mul(ctx0, gain, message));
+}
+
+ggml_tensor * llama_model_inkling2::graph::build_inputs() {
+ inp = build_inp_mem_hybrid_iswa_rel(hparams.n_relative_extent, hparams.n_swa);
+ attn = inp->get_attn();
+ recr = inp->get_recr();
- ggml_tensor * inp_pos = build_inp_pos();
- ggml_tensor * inp_out_ids = build_inp_out_ids();
+ ggml_tensor * inp_pos = build_inp_pos();
+ inp_out_ids = build_inp_out_ids();
// log-scaled positions: 1 + alpha*log(max(1, (pos + 1)/floor)), multiplies q and r of the full-attention layers
- ggml_tensor * tau = ggml_scale_bias(ctx0, ggml_cast(ctx0, inp_pos, GGML_TYPE_F32), 1.0f, 1.0f);
+ tau = ggml_scale_bias(ctx0, ggml_cast(ctx0, inp_pos, GGML_TYPE_F32), 1.0f, 1.0f);
tau = ggml_log(ctx0, ggml_clamp(ctx0, ggml_scale(ctx0, tau, 1.0f/hparams.n_log_scale_floor), 1.0f, INFINITY));
tau = ggml_reshape_3d(ctx0, ggml_scale_bias(ctx0, tau, hparams.f_log_scale_alpha, 1.0f), 1, 1, n_tokens);
@@ -256,134 +267,160 @@ llama_model_inkling2::graph::graph(const llm_graph_params & params) : llm_graph_
cur = build_norm(cur, model.tok_norm, nullptr, LLM_NORM_RMS, -1);
}
- std::vector<ggml_tensor *> blocks;
+ return cur;
+}
- for (int il = 0; il < n_layer; ++il) {
- const auto & layer = model.layers[il];
-
- const bool is_swa = hparams.is_swa(il);
- const int64_t n_head = hparams.n_head(il);
- const int64_t n_head_kv = hparams.n_head_kv(il);
- const int64_t n_embd_head = hparams.n_embd_head_k(il);
- const int64_t n_embd_k_gqa = n_embd_head*n_head_kv;
-
- ggml_tensor * state = recr->mctx->get_r_l(il);
- ggml_tensor * history = build_rs(recr, state, hparams.n_embd_r(), ubatch.n_seqs);
- history = ggml_reshape_3d(ctx0, history, hparams.n_shortconv_l_cache - 1, hparams.n_sconv_channels, ubatch.n_seqs);
-
- ggml_tensor * inpSA = cur;
-
- // attention inputs: SWA layers mix one, full-attention layers one each for Q, K and V
- ggml_tensor * inputs[3] = {cur, cur, cur};
- if (layer.attn_res_query) {
- ggml_tensor * mixed = mix(blocks, cur, layer.attn_res_query);
- for (int j = 0; j < 3; ++j) {
- inputs[j] = ggml_cont(ctx0, ggml_view_2d(ctx0, mixed, n_embd, n_tokens, mixed->nb[2], (is_swa ? 0 : j)*mixed->nb[1]));
- }
- }
+ggml_tensor * llama_model_inkling2::graph::build_layer(int il, int ip, ggml_tensor * cur, std::vector<ggml_tensor *> & blocks,
+ ggml_tensor * messages, const loop_layer * loop) {
+ GGML_ASSERT(!messages == !loop);
- if (is_swa) {
- inputs[0] = build_norm(inputs[0], layer.attn_norm, nullptr, LLM_NORM_RMS, il);
- inputs[1] = inputs[2] = inputs[0];
- } else {
- inputs[0] = build_norm(inputs[0], layer.attn_q_inp_norm, nullptr, LLM_NORM_RMS, il);
- inputs[1] = build_norm(inputs[1], layer.attn_k_inp_norm, nullptr, LLM_NORM_RMS, il);
- inputs[2] = build_norm(inputs[2], layer.attn_v_inp_norm, nullptr, LLM_NORM_RMS, il);
+ const auto & layer = model.layers[il];
+
+ const bool is_swa = hparams.is_swa(il);
+ const int64_t n_head = hparams.n_head(il);
+ const int64_t n_head_kv = hparams.n_head_kv(il);
+ const int64_t n_embd_head = hparams.n_embd_head_k(il);
+ const int64_t n_embd_k_gqa = n_embd_head*n_head_kv;
+
+ // the message of controller ctrl, [n_embd, n_tokens]
+ auto message = [&](int ctrl) {
+ return ggml_cont(ctx0, ggml_view_2d(ctx0, messages, n_embd, n_tokens, messages->nb[2], ctrl*messages->nb[1]));
+ };
+
+ ggml_tensor * state = recr->mctx->get_r_l(il);
+ ggml_tensor * history = build_rs(recr, state, hparams.n_embd_r(), ubatch.n_seqs);
+ history = ggml_reshape_3d(ctx0, history, hparams.n_shortconv_l_cache - 1, hparams.n_sconv_channels, ubatch.n_seqs);
+
+ ggml_tensor * inpSA = cur;
+
+ // attention inputs: SWA layers mix one, full-attention layers one each for Q, K and V
+ ggml_tensor * inputs[3] = {cur, cur, cur};
+ if (layer.attn_res_query) {
+ ggml_tensor * mixed = mix(blocks, cur, layer.attn_res_query);
+ for (int j = 0; j < 3; ++j) {
+ inputs[j] = ggml_cont(ctx0, ggml_view_2d(ctx0, mixed, n_embd, n_tokens, mixed->nb[2], (is_swa ? 0 : j)*mixed->nb[1]));
}
+ }
- // self-attention
- {
- ggml_tensor * Qcur = build_lora_mm(layer.wq, inputs[0]);
- ggml_tensor * Kcur = convolve(build_lora_mm(layer.wk, inputs[1]), layer.attn_k_conv,
- is_swa ? nullptr : build_lora_mm(layer.attn_k_conv_proj, inputs[1]), state, history, recr, 0);
- ggml_tensor * Vcur = convolve(build_lora_mm(layer.wv, inputs[2]), layer.attn_v_conv,
- is_swa ? nullptr : build_lora_mm(layer.attn_v_conv_proj, inputs[2]), state, history, recr, n_embd_k_gqa);
- ggml_tensor * Rcur = build_lora_mm(layer.attn_rel, inputs[0]);
+ if (messages) {
+ // the messages make the Q, K and V inputs of a SWA layer differ
+ for (int j = 0; j < 3; ++j) {
+ inputs[j] = control(inputs[j], message(LOOP_CTRL_Q + j), *loop, LOOP_CTRL_Q + j);
+ }
+ }
- Qcur = clip(ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), layer.attn_q_clip, hparams.f_qk_clip);
- Kcur = clip(ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), nullptr, hparams.f_qk_clip);
- Vcur = ggml_rms_norm(ctx0, ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens), hparams.f_norm_rms_eps);
- Rcur = clip(ggml_reshape_3d(ctx0, Rcur, hparams.n_relative, n_head, n_tokens), layer.attn_rel_clip, hparams.f_relative_clip);
+ if (is_swa && messages) {
+ for (int j = 0; j < 3; ++j) {
+ inputs[j] = build_norm(inputs[j], layer.attn_norm, nullptr, LLM_NORM_RMS, il);
+ }
+ } else if (is_swa) {
+ inputs[0] = build_norm(inputs[0], layer.attn_norm, nullptr, LLM_NORM_RMS, il);
+ inputs[1] = inputs[2] = inputs[0];
+ } else {
+ inputs[0] = build_norm(inputs[0], layer.attn_q_inp_norm, nullptr, LLM_NORM_RMS, il);
+ inputs[1] = build_norm(inputs[1], layer.attn_k_inp_norm, nullptr, LLM_NORM_RMS, il);
+ inputs[2] = build_norm(inputs[2], layer.attn_v_inp_norm, nullptr, LLM_NORM_RMS, il);
+ }
- if (!is_swa) {
- Qcur = ggml_mul(ctx0, Qcur, tau);
- Rcur = ggml_mul(ctx0, Rcur, tau);
- }
+ // self-attention
+ {
+ ggml_tensor * Qcur = build_lora_mm(layer.wq, inputs[0]);
+ ggml_tensor * Kcur = convolve(build_lora_mm(layer.wk, inputs[1]), layer.attn_k_conv,
+ is_swa ? nullptr : build_lora_mm(layer.attn_k_conv_proj, inputs[1]), state, history, recr, 0);
+ ggml_tensor * Vcur = convolve(build_lora_mm(layer.wv, inputs[2]), layer.attn_v_conv,
+ is_swa ? nullptr : build_lora_mm(layer.attn_v_conv_proj, inputs[2]), state, history, recr, n_embd_k_gqa);
+ ggml_tensor * Rcur = build_lora_mm(layer.attn_rel, inputs[0]);
+
+ Qcur = clip(ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), layer.attn_q_clip, hparams.f_qk_clip);
+ Kcur = clip(ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), nullptr, hparams.f_qk_clip);
+ Vcur = ggml_rms_norm(ctx0, ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens), hparams.f_norm_rms_eps);
+ Rcur = clip(ggml_reshape_3d(ctx0, Rcur, hparams.n_relative, n_head, n_tokens), layer.attn_rel_clip, hparams.f_relative_clip);
+
+ if (!is_swa) {
+ Qcur = ggml_mul(ctx0, Qcur, tau);
+ Rcur = ggml_mul(ctx0, Rcur, tau);
+ }
- cb(Qcur, "Qcur", il);
- cb(Kcur, "Kcur", il);
- cb(Vcur, "Vcur", il);
- cb(Rcur, "Rcur", il);
+ cb(Qcur, "Qcur", il);
+ cb(Kcur, "Kcur", il);
+ cb(Vcur, "Vcur", il);
+ cb(Rcur, "Rcur", il);
- ggml_tensor * rel_pos = is_swa ? inp->self_rel_pos_swa : inp->self_rel_pos;
+ ggml_tensor * rel_pos = is_swa ? inp->self_rel_pos_swa : inp->self_rel_pos;
- const int64_t n_stream = (is_swa ? attn->self_attn_mask_swa.raw : attn->self_attn_mask.raw)->ne[3];
- const int64_t n_kv = rel_pos->ne[0];
+ const int64_t n_stream = (is_swa ? attn->self_attn_mask_swa.raw : attn->self_attn_mask.raw)->ne[3];
+ const int64_t n_kv = rel_pos->ne[0];
- ggml_tensor * sinks = build_lora_mm(layer.attn_sink_proj, inputs[0]);
+ ggml_tensor * sinks = build_lora_mm(layer.attn_sink_proj, inputs[0]);
- // project r on the bias table first, then gather by distance: only the gathered bias is n_kv x n_tokens x n_head
- ggml_tensor * table = ggml_mul_mat(ctx0, ggml_pad(ctx0, layer.attn_rel_bias, 0, 1, 0, 0), Rcur);
+ // project r on the bias table first, then gather by distance: only the gathered bias is n_kv x n_tokens x n_head
+ ggml_tensor * table = ggml_mul_mat(ctx0, ggml_pad(ctx0, layer.attn_rel_bias, 0, 1, 0, 0), Rcur);
- // the sink logit t is per query: exp(s - t)/(1 + sum(exp(s - t))) = exp(s)/(exp(t) + sum(exp(s))), with a zero sink
- table = ggml_sub(ctx0, table, ggml_reshape_3d(ctx0, sinks, 1, n_head, n_tokens));
- table = ggml_cont(ctx0, ggml_permute(ctx0, table, 1, 0, 2, 3));
+ // the sink logit t is per query: exp(s - t)/(1 + sum(exp(s - t))) = exp(s)/(exp(t) + sum(exp(s))), with a zero sink
+ table = ggml_sub(ctx0, table, ggml_reshape_3d(ctx0, sinks, 1, n_head, n_tokens));
+ table = ggml_cont(ctx0, ggml_permute(ctx0, table, 1, 0, 2, 3));
- ggml_tensor * kq_b = ggml_get_rows(ctx0, table, rel_pos);
- kq_b = ggml_reshape_4d(ctx0, kq_b, n_head, n_kv, n_tokens/n_stream, n_stream);
- kq_b = ggml_cont(ctx0, ggml_permute(ctx0, kq_b, 2, 0, 1, 3));
- cb(kq_b, "kq_b", il);
+ ggml_tensor * kq_b = ggml_get_rows(ctx0, table, rel_pos);
+ kq_b = ggml_reshape_4d(ctx0, kq_b, n_head, n_kv, n_tokens/n_stream, n_stream);
+ kq_b = ggml_cont(ctx0, ggml_permute(ctx0, kq_b, 2, 0, 1, 3));
+ cb(kq_b, "kq_b", il);
- sinks = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_head), 0.0f);
+ sinks = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_head), 0.0f);
- cur = build_attn(inp,
- nullptr, nullptr, nullptr,
- Qcur, Kcur, Vcur, kq_b, sinks, 1.0f/sqrtf(n_embd_head), il);
+ cur = build_attn(inp,
+ nullptr, nullptr, nullptr,
+ Qcur, Kcur, Vcur, kq_b, sinks, 1.0f/sqrtf(n_embd_head), il);
- cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, build_lora_mm(layer.wqkv_gate, inputs[0])));
- cur = convolve(build_lora_mm(layer.wo, cur), layer.attn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa);
- cb(cur, "attn_out", il);
- }
+ cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, build_lora_mm(layer.wqkv_gate, inputs[0])));
+ cur = convolve(build_lora_mm(layer.wo, cur), layer.attn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa);
+ cb(cur, "attn_out", il);
+ }
- // the first layer of each block starts a new residual stream, the earlier one joins the mixed blocks
- if (il % hparams.attn_res_block_size == 0) {
- blocks.push_back(inpSA);
- } else {
- cur = ggml_add(ctx0, inpSA, cur);
- }
+ // the first layer of each block starts a new residual stream, the earlier one joins the mixed blocks
+ if (ip % hparams.attn_res_block_size == 0) {
+ blocks.push_back(inpSA);
+ } else {
+ cur = ggml_add(ctx0, inpSA, cur);
+ }
- ggml_tensor * ffn_inp = cur;
- cb(ffn_inp, "ffn_inp", il);
+ ggml_tensor * ffn_inp = cur;
+ cb(ffn_inp, "ffn_inp", il);
- // MoE
- cur = ggml_reshape_2d(ctx0, mix(blocks, cur, layer.ffn_res_query), n_embd, n_tokens);
- cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
-
- cur = build_moe_ffn(cur,
- layer.ffn_gate_inp,
- nullptr,
- nullptr,
- layer.ffn_down_exps,
- layer.ffn_exp_probs_b,
- hparams.n_expert,
- hparams.n_expert_used(il),
- LLM_FFN_SITU, true,
- hparams.expert_weights_scale,
- LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
- il,
- nullptr,
- layer.ffn_gate_up_exps,
- layer.ffn_up_exps_s,
- layer.ffn_gate_exps_s,
- layer.ffn_down_exps_s);
-
- cur = convolve(cur, layer.ffn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa + n_embd);
- cb(cur, "ffn_out", il);
-
- cur = ggml_add(ctx0, ffn_inp, cur);
- cb(cur, "l_out", il);
+ // MoE
+ cur = ggml_reshape_2d(ctx0, mix(blocks, cur, layer.ffn_res_query), n_embd, n_tokens);
+ if (messages) {
+ cur = control(cur, message(LOOP_CTRL_FFN), *loop, LOOP_CTRL_FFN);
}
+ cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
+
+ cur = build_moe_ffn(cur,
+ layer.ffn_gate_inp,
+ nullptr,
+ nullptr,
+ layer.ffn_down_exps,
+ layer.ffn_exp_probs_b,
+ hparams.n_expert,
+ hparams.n_expert_used(il),
+ LLM_FFN_SITU, true,
+ hparams.expert_weights_scale,
+ LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
+ il,
+ nullptr,
+ layer.ffn_gate_up_exps,
+ layer.ffn_up_exps_s,
+ layer.ffn_gate_exps_s,
+ layer.ffn_down_exps_s);
+
+ cur = convolve(cur, layer.ffn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa + n_embd);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, ffn_inp, cur);
+ cb(cur, "l_out", il);
+
+ return cur;
+}
+void llama_model_inkling2::graph::build_output(ggml_tensor * cur, const std::vector<ggml_tensor *> & blocks) {
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, model.output_res_query), n_embd, n_tokens);
cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
@@ -410,3 +447,14 @@ llama_model_inkling2::graph::graph(const llm_graph_params & params) : llm_graph_
cb(cur, "result_output", -1);
res->t_logits = cur;
}
+
+llama_model_inkling2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) {
+ ggml_tensor * cur = build_inputs();
+
+ std::vector<ggml_tensor *> blocks;
+ for (int il = 0; il < n_layer; ++il) {
+ cur = build_layer(il, il, cur, blocks);
+ }
+
+ build_output(cur, blocks);
+}
diff --git a/src/models/models.h b/src/models/models.h
index 822d707b9..636afeae9 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2492,6 +2492,19 @@ struct llama_model_inkling2 : public llama_model_base {
void load_arch_tensors(llama_model_loader & ml) override;
uint32_t graph_nodes_per_tensor() const override { return 32; }
+ // the layers stored in the file
+ virtual int n_layer_phys() const { return hparams.n_layer(); }
+
+ // message controllers of the q, k, v and ffn inputs of a looped layer (inkling2_loop)
+ enum { LOOP_CTRL_Q, LOOP_CTRL_K, LOOP_CTRL_V, LOOP_CTRL_FFN, LOOP_CTRL_COUNT };
+
+ struct loop_layer {
+ ggml_tensor * msg_query = nullptr;
+ ggml_tensor * ctrl_down[LOOP_CTRL_COUNT] = {};
+ ggml_tensor * ctrl_gain[LOOP_CTRL_COUNT] = {};
+ ggml_tensor * ctrl_forget[LOOP_CTRL_COUNT] = {};
+ };
+
struct graph : public llm_graph_context {
graph(const llm_graph_params & params);
ggml_tensor * mix(const std::vector<ggml_tensor *> & blocks, ggml_tensor * cur, ggml_tensor * query);
@@ -2504,6 +2517,52 @@ struct llama_model_inkling2 : public llama_model_base {
ggml_tensor * history,
llm_graph_input_rs * inp,
int64_t offset);
+ ggml_tensor * control(ggml_tensor * cur, ggml_tensor * message, const loop_layer & loop, int ctrl);
+
+ protected:
+ // the helpers alone, a derived graph builds its own body
+ struct no_build {};
+ graph(const llm_graph_params & params, no_build) : llm_graph_context(params) {}
+
+ // the inputs and the token embeddings
+ ggml_tensor * build_inputs();
+ // cache slot il of physical layer ip; messages [n_embd, LOOP_CTRL_COUNT, n_tokens] are mixed into its inputs
+ ggml_tensor * build_layer(int il, int ip, ggml_tensor * cur, std::vector<ggml_tensor *> & blocks,
+ ggml_tensor * messages = nullptr, const loop_layer * loop = nullptr);
+ void build_output(ggml_tensor * cur, const std::vector<ggml_tensor *> & blocks);
+
+ llm_graph_input_mem_hybrid_iswa_rel * inp = nullptr;
+ llm_graph_input_attn_kv_iswa * attn = nullptr;
+ llm_graph_input_rs * recr = nullptr;
+
+ ggml_tensor * inp_out_ids = nullptr;
+ ggml_tensor * tau = nullptr; // log-scaled positions
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+// the cache slots after the physical layers repeat the core [core_start, core_end) for each later pass
+struct llama_model_inkling2_loop : public llama_model_inkling2 {
+ llama_model_inkling2_loop(const struct llama_model_params & params) : llama_model_inkling2(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ int n_layer_phys() const override { return n_phys; }
+ int layer_phys(int il) const;
+
+ int n_phys = 0;
+ uint32_t core_start = 0;
+ uint32_t core_end = 0;
+ uint32_t n_passes = 0;
+ uint32_t gate_rank = 0;
+ bool varying = false; // trained for a varying number of passes
+
+ // indexed by physical layer - core_start
+ std::vector<loop_layer> loop_layers;
+
+ struct graph : public llama_model_inkling2::graph {
+ graph(const llm_graph_params & params);
};
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 90a32abc7..6faa8843d 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -326,7 +326,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
n_vocab = 3072; // TODO: should be 4096, but user code cannot get `n_vocab_out` yet [TAG_LLAMA_N_VOCAB_OUT]
} else if (arch == LLM_ARCH_HRM_TEXT) {
n_layer = 6; // 1 layer per stack x 2 h-cycles x (2 l-cycles + 1) cache slots
- } else if (arch == LLM_ARCH_INKLING2) {
+ } else if (arch == LLM_ARCH_INKLING2 || arch == LLM_ARCH_INKLING2_LOOP) {
n_embd = 128;
n_head = 4;
n_ff = 24;
@@ -703,7 +703,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
ms.add_kv(LLM_KV_MUDD_TAP_INDICES, std::vector<int32_t>({-1, -1, -1, -1, 0, 2, 1, 3}));
}
- if (arch == LLM_ARCH_INKLING2) {
+ if (arch == LLM_ARCH_INKLING2 || arch == LLM_ARCH_INKLING2_LOOP) {
+ const bool loop = arch == LLM_ARCH_INKLING2_LOOP;
ms.add_kv(LLM_KV_ATTENTION_RELATIVE_LENGTH, uint32_t(4));
ms.add_kv(LLM_KV_ATTENTION_RELATIVE_EXTENT, uint32_t(16));
ms.add_kv(LLM_KV_ATTENTION_QK_CLIP, 0.25f);
@@ -711,6 +712,14 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
ms.add_kv(LLM_KV_ATTENTION_LOG_SCALE_FLOOR, uint32_t(4));
ms.add_kv(LLM_KV_ATTENTION_LOG_SCALE_ALPHA, 0.1f);
ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(2));
+ if (loop) {
+ // prelude 0, core 1, coda 2; the loader adds slot 3 for the second pass over the core
+ ms.add_kv(LLM_KV_LOOP_START, uint32_t(1));
+ ms.add_kv(LLM_KV_LOOP_END, uint32_t(2));
+ ms.add_kv(LLM_KV_LOOP_COUNT, uint32_t(2));
+ ms.add_kv(LLM_KV_LOOP_GATE_RANK, uint32_t(8));
+ ms.add_kv(LLM_KV_LOOP_VARYING, false);
+ }
ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(4));
ms.add_kv(LLM_KV_VOCAB_SIZE, n_vocab + 2);
ms.add_kv(LLM_KV_VOCAB_SIZE_UNPADDED, n_vocab);
@@ -827,7 +836,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);
ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));
ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);
- ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12));
+ // the inkling2_loop core boundaries must be multiples of the block size
+ ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(arch == LLM_ARCH_INKLING2_LOOP ? 1 : 12));
ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f);
ms.add_kv(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, 25.0f);
ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
@@ -1145,6 +1155,7 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_METIS16:
case LLM_ARCH_ZONOS3:
case LLM_ARCH_INKLING2:
+ case LLM_ARCH_INKLING2_LOOP:
return true;
default:
return false;