Commit c622f623f for llama.cpp

commit c622f623fca9f0f085eb82f38874d1ab4a1791cd
Author: Xuan-Son Nguyen <son@huggingface.co>
Date:   Sat Nov 21 14:25:19 2026 +0100

    mtmd: support kyutai/hibiki realtime full-duplex (#32491)

    * mtmd: support hibiki realtime duplex

    * fix and clean up

    * fix long session quality

    * fix ci

    * tests : add the moshi fusion baseline for Metal

    The Metal fusion check found 5 RMS_NORM+MUL fusions in the new moshi
    arch and no baseline for them.

    ---------

    Co-authored-by: Pascal <admin@serveurperso.com>

diff --git a/conversion/__init__.py b/conversion/__init__.py
index 8869b7252..146c309e6 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -286,6 +286,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "Qwen3NextForCausalLM": "qwen",
     "Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
     "PocketTTSModel": "pockettts",
+    "MoshiModel": "moshi",
     "KokoroModel": "kokoro",
     "TinyVocaModel": "tiny_voca",
     "Zonos3Model": "zonos3",
@@ -432,6 +433,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "Qwen3ASRForConditionalGeneration": "qwen3vl",
     "Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
     "PocketTTSModel": "pockettts",
+    "MoshiModel": "moshi",
     "KokoroModel": "kokoro",
     "TinyVocaModel": "tiny_voca",
     "SopranoModel": "soprano",
diff --git a/conversion/moshi.py b/conversion/moshi.py
new file mode 100644
index 000000000..028319385
--- /dev/null
+++ b/conversion/moshi.py
@@ -0,0 +1,378 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Callable, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import ModelBase, MmprojModel, SentencePieceTokenTypes, TextModel, gguf, logger
+from .pockettts import mimi_seanet_tensors, mimi_tfm_tensors
+
+# Moshi family (Kyutai): a backbone over parallel streams (text + generated audio codes + input audio codes),
+# one position per 80 ms frame. Only Hibiki (simultaneous speech translation) is supported for now.
+# The checkpoint is not in the HF layout: a moshi-style config.json and two safetensors (LM + mimi).
+#
+# pipeline stage mapping:
+#   transformer + text_emb/text_linear  --> normal libllama text model (arch moshi)
+#   mimi encoder + RVQ + emb[dep_q:]    --> mtmd audio stream (projector moshi), rows of the input audio
+#   depformer + emb[:dep_q]             --> MTMD_GEN_PROCESS_TYPE_GEN_CODE, codes of a frame + their backbone embedding
+#   RVQ decode + mimi decoder           --> MTMD_GEN_PROCESS_TYPE_GEN_WAV
+#
+# Tricks:
+# - the initial text token (text_card) is appended to the vocab as <|text_start|>, the head has no logit for it;
+#   it only opens the stream, the embeddings of the initial codes of the generated audio are folded into it
+# - the condition of Hibiki ("very_good" description, added to every row) is folded into the text embeddings,
+#   every row has exactly one text embedding
+# - Hibiki ignores an early EOS: the reference gives it the embedding of PAD, done here too
+# - low-rank depformer embeddings are folded into full tables
+
+_MIMI_PREFIX = "mimi."
+_COND_VALUE = "very_good"
+
+
+def _read_config(dir_model: Path) -> dict[str, Any] | None:
+    path = dir_model / "config.json"
+    if not path.is_file():
+        return None
+    with open(path, "r", encoding="utf-8") as f:
+        config = json.load(f)
+    return config if isinstance(config, dict) and "moshi_name" in config else None
+
+
+def _tensor_shape(path: Path, name: str) -> tuple[int, ...]:
+    with gguf.utility.SafetensorsLocal(path) as part:
+        return tuple(part[name].shape)
+
+
+@ModelBase.register_hparams_loader(lambda dir_model: _read_config(dir_model) is not None)
+def _load_hparams(dir_model: Path) -> dict[str, Any]:
+    config = _read_config(dir_model)
+    assert config is not None
+    model_type = config.get("model_type", "moshi")
+    if model_type != "hibiki":
+        raise NotImplementedError(f"moshi checkpoint of type {model_type!r} is not supported yet")
+    logger.info(f"gguf: detected a moshi checkpoint ({model_type})")
+
+    # mtmd has the layout of Hibiki: the acoustic codes lag by 2 frames, the depformer has no positional embedding,
+    # its steps past the last weight set reuse it, and the initial code is the extra entry after the mimi codebook
+    n_q = config["dep_q"]
+    schedule = config.get("depformer_weights_per_step_schedule") or list(range(n_q))
+    # text, then the generated and the input codes
+    if config["delays"] != [0] + ([0] + [2] * (n_q - 1)) * 2:
+        raise NotImplementedError(f"unsupported delays {config['delays']}")
+    if schedule != [min(i, max(schedule)) for i in range(n_q)] or config.get("depformer_pos_emb", "none") != "none":
+        raise NotImplementedError("unsupported depformer layout")
+    if config["card"] != 2048 or config["n_q"] != 2 * n_q:
+        raise NotImplementedError("unsupported audio codebooks")
+
+    lm_path = dir_model / config["moshi_name"]
+    # the gating hidden size is derived from dim_feedforward, read it back from the tensors
+    n_ff     = _tensor_shape(lm_path, "transformer.layers.0.gating.linear_out.weight")[1]
+    n_ff_dep = _tensor_shape(lm_path, "depformer.layers.0.gating.0.linear_out.weight")[1]
+    return {
+        "architectures": ["MoshiModel"],
+        "model_type": "moshi",
+        "moshi_config": config,
+        "num_hidden_layers": config["num_layers"],
+        "hidden_size": config["dim"],
+        "intermediate_size": n_ff,
+        "num_attention_heads": config["num_heads"],
+        "num_key_value_heads": config["num_heads"],
+        "max_position_embeddings": config["context"],
+        "sliding_window": config["context"],
+        "rope_theta": config["max_period"],
+        "rms_norm_eps": 1e-8, # rms_norm_f32
+        "vocab_size": config["text_card"] + 1,
+        "audio_config": {
+            # mimi encoder transformer, constant across the family
+            "num_hidden_layers": 8,
+            "hidden_size": 512,
+            "intermediate_size": 2048,
+            "num_attention_heads": 8,
+        },
+        "depformer_config": {
+            "num_hidden_layers": config["depformer_num_layers"],
+            "hidden_size": config["depformer_dim"],
+            "intermediate_size": n_ff_dep,
+            "num_attention_heads": config["depformer_num_heads"],
+        },
+    }
+
+
+# the LM and mimi weights are in two files named by the config, mimi tensors get the "mimi." prefix
+def _moshi_config(model: ModelBase) -> dict[str, Any]:
+    config = model.hparams.get("moshi_config")
+    if config is None and isinstance(model, MmprojModel):
+        config = model.global_config["moshi_config"] # the mmproj moves the hparams aside
+    assert config is not None
+    return config
+
+
+def _index_moshi_tensors(model: ModelBase) -> dict[str, Callable[[], Tensor]]:
+    config = _moshi_config(model)
+    tensors = model.index_local_tensors(Path(config["moshi_name"]).stem)
+    for name, gen in model.index_local_tensors(Path(config["mimi_name"]).stem).items():
+        tensors[_MIMI_PREFIX + name] = gen
+    return tensors
+
+
+def _low_rank_embd(model: ModelBase, prefix: str) -> Tensor:
+    embd = model.model_tensors[prefix + ".weight"]().float()
+    proj = model.model_tensors.get(prefix + ".low_rank.weight")
+    return embd @ proj().float().T if proj is not None else embd
+
+
+def _condition(model: ModelBase, value: str) -> Tensor:
+    # LUT conditioner: output_proj(embed[index of value])
+    lut = _moshi_config(model)["conditioners"]["description"]["lut"]
+    prefix = "condition_provider.conditioners.description."
+    embed = model.model_tensors[prefix + "embed.weight"]().float()
+    proj  = model.model_tensors[prefix + "output_proj.weight"]().float()
+    return embed[lut["possible_values"].index(value)] @ proj.T
+
+
+@ModelBase.register("MoshiModel")
+@ModelBase.example("kyutai/hibiki-1b-pytorch-bf16", "kyutai/hibiki-2b-pytorch-bf16")
+class MoshiModel(TextModel):
+    model_arch = gguf.MODEL_ARCH.MOSHI
+
+    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
+        assert remote_hf_model_id is None
+        return _index_moshi_tensors(self)
+
+    def set_vocab(self):
+        from sentencepiece import SentencePieceProcessor
+
+        config = _moshi_config(self)
+        sp = SentencePieceProcessor()
+        sp.LoadFromFile(str(self.dir_model / config["tokenizer_name"]))
+        assert sp.vocab_size() == config["text_card"]
+
+        tokens: list[bytes] = []
+        scores: list[float] = []
+        toktypes: list[int] = []
+        for i in range(sp.vocab_size()):
+            toktype = SentencePieceTokenTypes.NORMAL
+            if sp.IsControl(i) or sp.IsUnknown(i):
+                # <unk> (0) is the end-of-padding marker of the text stream, it has no text
+                toktype = SentencePieceTokenTypes.CONTROL
+            elif sp.IsUnused(i):
+                toktype = SentencePieceTokenTypes.UNUSED
+            elif sp.IsByte(i):
+                toktype = SentencePieceTokenTypes.BYTE
+            tokens.append(sp.IdToPiece(i).encode("utf-8"))
+            scores.append(sp.GetScore(i))
+            toktypes.append(toktype)
+
+        # the initial token of the text stream
+        tokens.append(b"<|text_start|>")
+        scores.append(-1000.0)
+        toktypes.append(SentencePieceTokenTypes.CONTROL)
+
+        self.gguf_writer.add_tokenizer_model("llama")
+        self.gguf_writer.add_tokenizer_pre("default")
+        self.gguf_writer.add_token_list(tokens)
+        self.gguf_writer.add_token_scores(scores)
+        self.gguf_writer.add_token_types(toktypes)
+        self.gguf_writer.add_unk_token_id(sp.unk_id())
+        self.gguf_writer.add_bos_token_id(sp.bos_id())
+        self.gguf_writer.add_eos_token_id(sp.eos_id())
+        self.gguf_writer.add_pad_token_id(sp.pad_id())
+        self.gguf_writer.add_add_bos_token(False)
+        self.gguf_writer.add_add_eos_token(False)
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
+
+        # sampling of the text tokens, the audio codes are sampled by mtmd
+        gen = _moshi_config(self).get("lm_gen_config", {})
+        self.gguf_writer.add_sampling_temp(gen.get("temp_text", 0.7))
+        self.gguf_writer.add_sampling_top_k(gen.get("top_k_text", 25))
+        self.gguf_writer.add_sampling_top_p(1.0)
+        self.gguf_writer.add_sampling_min_p(0.0)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        T = gguf.MODEL_TENSOR
+
+        if name == "text_emb.weight":
+            config = _moshi_config(self)
+            embd = data_torch.float().clone()
+            embd[2] = embd[3] # EOS -> PAD, see get_moshi() of the reference
+            embd += _condition(self, _COND_VALUE)
+            assert embd.shape[0] == config["text_card"] + 1
+            # <|text_start|> only opens the stream, with the initial codes of the generated audio
+            for k in range(config["dep_q"]):
+                embd[-1] += self.model_tensors[f"emb.{k}.weight"]()[config["card"]].float()
+            yield (self.format_tensor_name(T.TOKEN_EMBD), embd)
+            return
+        if name == "text_linear.weight":
+            yield (self.format_tensor_name(T.OUTPUT), data_torch)
+            return
+        if name == "out_norm.alpha":
+            yield (self.format_tensor_name(T.OUTPUT_NORM), data_torch.reshape(-1))
+            return
+
+        if not name.startswith("transformer.layers."):
+            return # mimi, depformer and the audio embeddings go to the mmproj
+        assert bid is not None
+        key = name.split(f"layers.{bid}.", 1)[1]
+
+        if key == "self_attn.in_proj_weight":
+            q, k, v = data_torch.chunk(3, dim=0)
+            yield (self.format_tensor_name(T.ATTN_Q, bid), q)
+            yield (self.format_tensor_name(T.ATTN_K, bid), k)
+            yield (self.format_tensor_name(T.ATTN_V, bid), v)
+        elif key == "self_attn.out_proj.weight":
+            yield (self.format_tensor_name(T.ATTN_OUT, bid), data_torch)
+        elif key == "norm1.alpha":
+            yield (self.format_tensor_name(T.ATTN_NORM, bid), data_torch.reshape(-1))
+        elif key == "norm2.alpha":
+            yield (self.format_tensor_name(T.FFN_NORM, bid), data_torch.reshape(-1))
+        elif key == "gating.linear_in.weight":
+            # silu(first half) * second half
+            gate, up = data_torch.chunk(2, dim=0)
+            yield (self.format_tensor_name(T.FFN_GATE, bid), gate)
+            yield (self.format_tensor_name(T.FFN_UP, bid), up)
+        elif key == "gating.linear_out.weight":
+            yield (self.format_tensor_name(T.FFN_DOWN, bid), data_torch)
+        else:
+            raise ValueError(f"unexpected tensor {name}")
+
+
+@ModelBase.register("MoshiModel")
+@ModelBase.example("kyutai/hibiki-1b-pytorch-bf16", "kyutai/hibiki-2b-pytorch-bf16")
+class MoshiMmprojModel(MmprojModel):
+    has_audio_encoder = True
+    has_vision_encoder = False
+
+    def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
+        assert remote_hf_model_id is None
+        return _index_moshi_tensors(self)
+
+    def _n_codebooks(self) -> int:
+        return _moshi_config(self)["dep_q"]
+
+    def set_gguf_parameters(self):
+        self.gguf_writer.add_file_type(self.ftype)
+        assert self.hparams_audio is not None
+        dep = self.global_config["depformer_config"]
+
+        # input audio: mimi encoder + RVQ + embeddings of the input codes
+        self.gguf_writer.add_clip_has_audio_encoder(True)
+        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.MOSHI)
+        self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
+        self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"])
+        self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"])
+        self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"])
+        self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"])
+        self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
+        # mimi convolves the waveform directly, it is passed around as a 1-row "mel"
+        self.gguf_writer.add_audio_num_mel_bins(1)
+
+        # generation: depformer + mimi decoder, the mimi hparams are constant and held by mtmd
+        self.gguf_writer.add_clip_has_gen_audio_encoder(True)
+        self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.MOSHI_GEN)
+        self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
+        self.gguf_writer.add_gen_audio_embedding_length(dep["hidden_size"])
+        self.gguf_writer.add_gen_audio_feed_forward_length(dep["intermediate_size"])
+        self.gguf_writer.add_gen_audio_block_count(dep["num_hidden_layers"])
+        self.gguf_writer.add_gen_audio_head_count(dep["num_attention_heads"])
+        self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-8)
+        self.gguf_writer.add_gen_audio_model_variant(_moshi_config(self)["model_type"])
+
+    def tensor_force_quant(self, name, new_name, bid, n_dims):
+        del name, bid, n_dims
+        # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path
+        if ".seanet." in new_name or new_name in ("a.downsample.conv.weight", "a.gen.wav.upsample.weight"):
+            return gguf.GGMLQuantizationType.F16
+        # nearest-code search and RVQ decode stay exact
+        if ".codebook" in new_name or new_name.startswith("a.codec.vq."):
+            return gguf.GGMLQuantizationType.F32
+        return False
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        del bid
+        T = gguf.MODEL_TENSOR
+        if name.startswith(("a.", "mm.a.")):
+            yield (name, data_torch) # already named by generate_extra_tensors()
+            return
+        if not name.startswith(_MIMI_PREFIX):
+            return # the LM tensors are stacked in generate_extra_tensors()
+        if name == "mimi.downsample.conv.conv.conv.weight":
+            yield (self.format_tensor_name(T.A_ENC_DOWNSAMPLE_CONV), data_torch)
+        elif name == "mimi.upsample.convtr.convtr.convtr.weight":
+            yield (self.format_tensor_name(T.A_GEN_WAV_UPSAMPLE), data_torch)
+        elif "_transformer.transformer.layers." in name:
+            # pocket-tts names it in_proj.weight
+            yield from mimi_tfm_tensors(self, name.replace(".in_proj_weight", ".in_proj.weight"), data_torch)
+        elif name.startswith(("mimi.encoder.model.", "mimi.decoder.model.")):
+            yield from mimi_seanet_tensors(self, name, data_torch, 4)
+
+    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+        T = gguf.MODEL_TENSOR
+        config = _moshi_config(self)
+        n_q = self._n_codebooks()
+
+        def get(name: str) -> Tensor:
+            return self.model_tensors[name]()
+
+        def stack(names: Iterable[str]) -> Tensor:
+            return torch.stack([get(n) for n in names], dim=0)
+
+        # mimi RVQ: one semantic codebook, then n_q - 1 acoustic ones
+        def codebooks(prefix: str, n: int) -> Tensor:
+            books = []
+            for i in range(n):
+                embedding_sum = get(f"{prefix}.vq.layers.{i}._codebook.embedding_sum").float()
+                cluster_usage = get(f"{prefix}.vq.layers.{i}._codebook.cluster_usage").float()
+                books.append(embedding_sum / cluster_usage.clamp_min(1e-5).unsqueeze(-1))
+            return torch.stack(books, dim=0) if n > 1 else books[0]
+
+        q = "mimi.quantizer."
+        cb_first = codebooks(q + "rvq_first", 1)
+        cb_rest  = codebooks(q + "rvq_rest", n_q - 1)
+        yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN), get(q + "rvq_first.input_proj.weight").squeeze(-1))
+        yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_CB), cb_first)
+        yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN), get(q + "rvq_rest.input_proj.weight").squeeze(-1))
+        yield (self.format_tensor_name(T.A_CODEC_VQ_REST_CB), cb_rest)
+        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_OUT), get(q + "rvq_first.output_proj.weight").squeeze(-1))
+        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), cb_first)
+        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_OUT), get(q + "rvq_rest.output_proj.weight").squeeze(-1))
+        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_CB), cb_rest)
+
+        # backbone embeddings of the audio codes: the generated stream first, then the input stream
+        assert config["n_q"] == 2 * n_q
+        yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), stack(f"emb.{k}.weight" for k in range(n_q)))
+        yield (self.format_tensor_name(T.A_MM_CODE_EMBD), stack(f"emb.{k}.weight" for k in range(n_q, 2 * n_q)))
+
+        # depformer: per-step weights are stacked on the last ggml dim, steps past the last weight set reuse it
+        dep = self.global_config["depformer_config"]
+        n_dim = dep["hidden_size"]
+        n_in = sum(1 for name in self.model_tensors if name.startswith("depformer_in."))
+        yield (self.format_tensor_name(T.A_GEN_CODE_PROJ_IN), stack(f"depformer_in.{i}.weight" for i in range(n_in)))
+        yield (self.format_tensor_name(T.A_GEN_CODE_TOK_EMBD), _low_rank_embd(self, "depformer_text_emb"))
+        yield (self.format_tensor_name(T.A_GEN_CODE_EMBD),
+               torch.stack([_low_rank_embd(self, f"depformer_emb.{i}") for i in range(n_q - 1)], dim=0))
+        yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), stack(f"linears.{k}.weight" for k in range(n_q)))
+
+        for il in range(dep["num_hidden_layers"]):
+            p = f"depformer.layers.{il}."
+            n_w = sum(1 for name in self.model_tensors if name.startswith(p + "gating.") and name.endswith(".linear_in.weight"))
+            assert n_w == n_in, "depformer_in and the per-step weights use the same schedule"
+            qkv = get(p + "self_attn.in_proj_weight").reshape(n_w, 3, n_dim, n_dim)
+            yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, il), qkv[:, 0])
+            yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, il), qkv[:, 1])
+            yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, il), qkv[:, 2])
+            yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_OUT, il), get(p + "self_attn.out_proj.weight").reshape(n_w, n_dim, n_dim))
+            gate_up = stack(f"{p}gating.{i}.linear_in.weight" for i in range(n_w))
+            gate, up = gate_up.chunk(2, dim=1)
+            yield (self.format_tensor_name(T.A_GEN_CODE_FFN_GATE, il), gate)
+            yield (self.format_tensor_name(T.A_GEN_CODE_FFN_UP, il), up)
+            yield (self.format_tensor_name(T.A_GEN_CODE_FFN_DOWN, il), stack(f"{p}gating.{i}.linear_out.weight" for i in range(n_w)))
+            yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_NORM, il), get(p + "norm1.alpha").reshape(-1))
+            yield (self.format_tensor_name(T.A_GEN_CODE_FFN_NORM, il), get(p + "norm2.alpha").reshape(-1))
diff --git a/conversion/pockettts.py b/conversion/pockettts.py
index b0cddc685..4a3d980cd 100644
--- a/conversion/pockettts.py
+++ b/conversion/pockettts.py
@@ -271,26 +271,84 @@ class PocketTTSModel(TextModel):
         return torch.cat(rows, dim=0)


+# mimi tensors (shared with moshi): the transformers of the encoder and the decoder, and the SEANet stacks
+MIMI_TFM_MAP = {
+    "norm1":              (gguf.MODEL_TENSOR.A_ENC_INPUT_NORM,  gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM),
+    "norm2":              (gguf.MODEL_TENSOR.A_ENC_OUTPUT_NORM, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM),
+    "self_attn.out_proj": (gguf.MODEL_TENSOR.A_ENC_OUTPUT,      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT),
+    "linear1":            (gguf.MODEL_TENSOR.A_ENC_FFN_UP,      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP),
+    "linear2":            (gguf.MODEL_TENSOR.A_ENC_FFN_DOWN,    gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN),
+    "layer_scale_1.scale": (gguf.MODEL_TENSOR.A_ENC_ATTN_SCALE, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_SCALE),
+    "layer_scale_2.scale": (gguf.MODEL_TENSOR.A_ENC_FFN_SCALE_LS, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE),
+}
+MIMI_TFM_QKV = (
+    (gguf.MODEL_TENSOR.A_ENC_ATTN_Q, gguf.MODEL_TENSOR.A_ENC_ATTN_K, gguf.MODEL_TENSOR.A_ENC_ATTN_V),
+    (gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V),
+)
+
+
+def mimi_tfm_tensors(model: MmprojModel, name: str, data_torch: Tensor) -> Iterable[tuple[str, Tensor]]:
+    is_decoder = name.startswith("mimi.decoder_transformer.")
+    bid = int(name.split("_transformer.transformer.layers.", 1)[1].split(".")[0])
+    key_with_suffix = name.split(f".layers.{bid}.", 1)[1]
+
+    if key_with_suffix == "self_attn.in_proj.weight":
+        q, k, v = data_torch.chunk(3, dim=0)
+        names = MIMI_TFM_QKV[1 if is_decoder else 0]
+        for tensor, part in zip(names, (q, k, v)):
+            yield (model.format_tensor_name(tensor, bid), part)
+        return
+
+    key, suffix = key_with_suffix.rsplit(".", 1)
+    entry = MIMI_TFM_MAP.get(key) or MIMI_TFM_MAP.get(key_with_suffix)
+    if entry is None:
+        return
+    tensor = entry[1 if is_decoder else 0]
+    suffix = ".weight" if key_with_suffix.endswith(".scale") else "." + suffix
+    yield (model.format_tensor_name(tensor, bid, suffix=suffix), data_torch)
+
+
+def mimi_seanet_tensors(model: MmprojModel, name: str, data_torch: Tensor, n_stages: int) -> Iterable[tuple[str, Tensor]]:
+    T = gguf.MODEL_TENSOR
+    is_decoder = name.startswith("mimi.decoder.")
+    idx = int(name.split(".model.", 1)[1].split(".")[0])
+    suffix = "." + name.rsplit(".", 1)[1]
+
+    conv_in, conv_out, res1, res2, scale = (
+        (T.A_GEN_WAV_SEANET_CONV_IN, T.A_GEN_WAV_SEANET_CONV_OUT, T.A_GEN_WAV_SEANET_RES_CONV1,
+         T.A_GEN_WAV_SEANET_RES_CONV2, T.A_GEN_WAV_SEANET_SCALE_CONV)
+        if is_decoder else
+        (T.A_ENC_SEANET_CONV_IN, T.A_ENC_SEANET_CONV_OUT, T.A_ENC_SEANET_RES_CONV1,
+         T.A_ENC_SEANET_RES_CONV2, T.A_ENC_SEANET_SCALE_CONV)
+    )
+
+    if idx == 0:
+        yield (model.format_tensor_name(conv_in, suffix=suffix), data_torch)
+        return
+    if idx == 3 * n_stages + 2:
+        yield (model.format_tensor_name(conv_out, suffix=suffix), data_torch)
+        return
+
+    for stage in range(n_stages):
+        res_idx = _DEC_RES_IDX(stage) if is_decoder else _ENC_RES_IDX(stage)
+        scale_idx = _DEC_SCALE_IDX(stage) if is_decoder else _ENC_SCALE_IDX(stage)
+        if idx == scale_idx:
+            yield (model.format_tensor_name(scale, stage, suffix=suffix), data_torch)
+            return
+        if idx == res_idx:
+            # block.1 is the dilated conv, block.3 the pointwise one (0 and 2 are ELU)
+            inner = int(name.split(".block.", 1)[1].split(".")[0])
+            tensor = res1 if inner == 1 else res2
+            yield (model.format_tensor_name(tensor, stage, suffix=suffix), data_torch)
+            return
+
+
 @ModelBase.register("PocketTTSModel")
 # [TAG_HF_EXAMPLE_MISSING] model is gated, and the checkpoint requires cd to subdir, not supported here
 class PocketTTSMmprojModel(MmprojModel):
     has_audio_encoder = True
     has_vision_encoder = False

-    _MIMI_TFM_MAP = {
-        "norm1":              (gguf.MODEL_TENSOR.A_ENC_INPUT_NORM,  gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM),
-        "norm2":              (gguf.MODEL_TENSOR.A_ENC_OUTPUT_NORM, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM),
-        "self_attn.out_proj": (gguf.MODEL_TENSOR.A_ENC_OUTPUT,      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT),
-        "linear1":            (gguf.MODEL_TENSOR.A_ENC_FFN_UP,      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP),
-        "linear2":            (gguf.MODEL_TENSOR.A_ENC_FFN_DOWN,    gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN),
-        "layer_scale_1.scale": (gguf.MODEL_TENSOR.A_ENC_ATTN_SCALE, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_SCALE),
-        "layer_scale_2.scale": (gguf.MODEL_TENSOR.A_ENC_FFN_SCALE_LS, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE),
-    }
-    _MIMI_TFM_QKV = (
-        (gguf.MODEL_TENSOR.A_ENC_ATTN_Q, gguf.MODEL_TENSOR.A_ENC_ATTN_K, gguf.MODEL_TENSOR.A_ENC_ATTN_V),
-        (gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V),
-    )
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.has_audio_encoder = _has_voice_encoder(self.dir_model)
@@ -419,11 +477,11 @@ class PocketTTSMmprojModel(MmprojModel):
             return

         if "_transformer.transformer.layers." in name:
-            yield from self._mimi_tfm_tensor(name, data_torch)
+            yield from mimi_tfm_tensors(self, name, data_torch)
             return

         if name.startswith("mimi.encoder.model.") or name.startswith("mimi.decoder.model."):
-            yield from self._seanet_tensor(name, data_torch)
+            yield from mimi_seanet_tensors(self, name, data_torch, _N_SEANET_STAGES)
             return

         return
@@ -471,57 +529,3 @@ class PocketTTSMmprojModel(MmprojModel):
             if tensor is not None:
                 yield (self.format_tensor_name(tensor, bid, suffix=suffix), data_torch)
             return
-
-    def _mimi_tfm_tensor(self, name: str, data_torch: Tensor) -> Iterable[tuple[str, Tensor]]:
-        is_decoder = name.startswith("mimi.decoder_transformer.")
-        bid = int(name.split("_transformer.transformer.layers.", 1)[1].split(".")[0])
-        key_with_suffix = name.split(f".layers.{bid}.", 1)[1]
-
-        if key_with_suffix == "self_attn.in_proj.weight":
-            q, k, v = data_torch.chunk(3, dim=0)
-            names = self._MIMI_TFM_QKV[1 if is_decoder else 0]
-            for tensor, part in zip(names, (q, k, v)):
-                yield (self.format_tensor_name(tensor, bid), part)
-            return
-
-        key, suffix = key_with_suffix.rsplit(".", 1)
-        entry = self._MIMI_TFM_MAP.get(key) or self._MIMI_TFM_MAP.get(key_with_suffix)
-        if entry is None:
-            return
-        tensor = entry[1 if is_decoder else 0]
-        suffix = ".weight" if key_with_suffix.endswith(".scale") else "." + suffix
-        yield (self.format_tensor_name(tensor, bid, suffix=suffix), data_torch)
-
-    def _seanet_tensor(self, name: str, data_torch: Tensor) -> Iterable[tuple[str, Tensor]]:
-        T = gguf.MODEL_TENSOR
-        is_decoder = name.startswith("mimi.decoder.")
-        idx = int(name.split(".model.", 1)[1].split(".")[0])
-        suffix = "." + name.rsplit(".", 1)[1]
-
-        conv_in, conv_out, res1, res2, scale = (
-            (T.A_GEN_WAV_SEANET_CONV_IN, T.A_GEN_WAV_SEANET_CONV_OUT, T.A_GEN_WAV_SEANET_RES_CONV1,
-             T.A_GEN_WAV_SEANET_RES_CONV2, T.A_GEN_WAV_SEANET_SCALE_CONV)
-            if is_decoder else
-            (T.A_ENC_SEANET_CONV_IN, T.A_ENC_SEANET_CONV_OUT, T.A_ENC_SEANET_RES_CONV1,
-             T.A_ENC_SEANET_RES_CONV2, T.A_ENC_SEANET_SCALE_CONV)
-        )
-
-        if idx == 0:
-            yield (self.format_tensor_name(conv_in, suffix=suffix), data_torch)
-            return
-        if idx == 3 * _N_SEANET_STAGES + 2:
-            yield (self.format_tensor_name(conv_out, suffix=suffix), data_torch)
-            return
-
-        for stage in range(_N_SEANET_STAGES):
-            res_idx = _DEC_RES_IDX(stage) if is_decoder else _ENC_RES_IDX(stage)
-            scale_idx = _DEC_SCALE_IDX(stage) if is_decoder else _ENC_SCALE_IDX(stage)
-            if idx == scale_idx:
-                yield (self.format_tensor_name(scale, stage, suffix=suffix), data_torch)
-                return
-            if idx == res_idx:
-                # block.1 is the dilated conv, block.3 the pointwise one (0 and 2 are ELU)
-                inner = int(name.split(".block.", 1)[1].split(".")[0])
-                tensor = res1 if inner == 1 else res2
-                yield (self.format_tensor_name(tensor, stage, suffix=suffix), data_torch)
-                return
diff --git a/docs/multimodal/tts.md b/docs/multimodal/tts.md
index e009566c0..195c19fb7 100644
--- a/docs/multimodal/tts.md
+++ b/docs/multimodal/tts.md
@@ -15,6 +15,7 @@ Supported models:
 - [Fish Audio S2 Pro](#fish-audio-s2-pro)
 - [YuE2](#yue2) (songs)
 - [MiniMax Music 3](#minimax-music-3) (songs)
+- [Hibiki](#hibiki) (speech translation, `llama-server` realtime endpoint only)
 - [Kokoro](#kokoro)
 - [tiny-voca](#tiny-voca)
 - [VoxCPM2](#voxcpm2)
@@ -367,6 +368,22 @@ python convert_hf_to_gguf.py MiniMax-Music3 --mmproj --outfile mmproj-minimax-mu

 The mmproj holds the depth decoder and the DiT layers (in the `--outtype` type), and the condition encoder, the small DiT projections and the DAC decoder, always in F32.

+## Hibiki
+
+Hibiki (Kyutai) translates French speech to English speech in real time. It is a full duplex model: it only runs through the [realtime endpoint](../../tools/server/README.md#full-duplex) of `llama-server`, the audio goes in and comes out every 80 ms.
+
+The checkpoint is not in the HF layout, the converter reads its `config.json` and the LM and mimi files it names (Hibiki-1B `kyutai/hibiki-1b-pytorch-bf16`, Hibiki-2B `kyutai/hibiki-2b-pytorch-bf16`):
+
+```sh
+python convert_hf_to_gguf.py path/to/hibiki-1b-pytorch-bf16 --outfile hibiki-1b.gguf
+python convert_hf_to_gguf.py path/to/hibiki-1b-pytorch-bf16 --mmproj --outfile mmproj-hibiki-1b.gguf
+
+llama-server -m hibiki-1b.gguf --mmproj mmproj-hibiki-1b.gguf -c 1500
+```
+
+- The voice of the output is conditioned like the reference (`very_good`), classifier-free guidance is not supported yet
+- The audio codes are sampled with the settings of the checkpoint (temperature 0.8, top-k 250), so is the text (temperature 0.8, top-k 50) unless the server sampling options override it
+
 ## Kokoro

 Available params:
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index cc0347bba..0592f5e01 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -778,6 +778,7 @@ class MODEL_ARCH(IntEnum):
     NANBEIGE         = auto()
     QWEN3TTS         = auto()
     POCKETTTS        = auto()
+    MOSHI            = auto()
     ZONOS3           = auto()
     KOKORO           = auto()
     TINY_VOCA        = auto()
@@ -1793,6 +1794,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.NANBEIGE:         "nanbeige",
     MODEL_ARCH.QWEN3TTS:         "qwen3tts",
     MODEL_ARCH.POCKETTTS:        "pockettts",
+    MODEL_ARCH.MOSHI:            "moshi",
     MODEL_ARCH.ZONOS3:           "zonos3",
     MODEL_ARCH.KOKORO:           "kokoro",
     MODEL_ARCH.TINY_VOCA:        "tiny-voca",
@@ -6957,6 +6959,20 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_DOWN,
         MODEL_TENSOR.FFN_UP,
     ],
+    MODEL_ARCH.MOSHI: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+    ],
     MODEL_ARCH.TINY_VOCA: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,
@@ -7514,6 +7530,8 @@ class VisionProjectorType:
     QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor
     POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder
     POCKETTTS_GEN = "pockettts_gen" # audio generation: flow-matching decoder + mimi decoder
+    MOSHI = "moshi" # audio: streaming mimi encoder, input audio codes -> backbone rows
+    MOSHI_GEN = "moshi_gen" # audio generation: depformer + mimi decoder
     CHATTERBOX_SPKENC = "chatterbox_spkenc" # audio: CAMPPlus speaker encoder
     CHATTERBOX = "chatterbox" # audio generation: s3gen flow matching + HiFT vocoder
     SOPRANO = "soprano" # audio generation: Vocos decoder
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 3cda4f7fb..cdb1f3794 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -176,6 +176,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_NANBEIGE,         "nanbeige"         },
     { LLM_ARCH_QWEN3TTS,         "qwen3tts"         },
     { LLM_ARCH_POCKETTTS,        "pockettts"        },
+    { LLM_ARCH_MOSHI,            "moshi"            },
     { LLM_ARCH_ZONOS3,           "zonos3"           },
     { LLM_ARCH_KOKORO,           "kokoro"           },
     { LLM_ARCH_TINY_VOCA,        "tiny-voca"        },
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 1ac22ecee..6517a3ea7 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -177,6 +177,7 @@ enum llm_arch {
     LLM_ARCH_NANBEIGE,
     LLM_ARCH_QWEN3TTS,
     LLM_ARCH_POCKETTTS,
+    LLM_ARCH_MOSHI,
     LLM_ARCH_ZONOS3,
     LLM_ARCH_KOKORO,
     LLM_ARCH_TINY_VOCA,
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 81e28b1c5..11c456947 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -126,6 +126,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_qwen3tts(params);
         case LLM_ARCH_POCKETTTS:
             return new llama_model_pockettts(params);
+        case LLM_ARCH_MOSHI:
+            return new llama_model_moshi(params);
         case LLM_ARCH_ZONOS3:
             return new llama_model_zonos3(params);
         case LLM_ARCH_KOKORO:
@@ -3154,6 +3156,27 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                     /* filter_recr       */ std::move(filter_recr),
                     /* filter_idx        */ std::move(filter_idx));
             } break;
+        case LLM_ARCH_MOSHI:
+            {
+                // every layer attends to the same sliding window: one cache that masks it, the cells past the window are reused
+                res = new llama_kv_cache(
+                        *this,
+                        params.no_alloc,
+                        hparams,
+                        params.type_k,
+                        params.type_v,
+                        !cparams.flash_attn,
+                        cparams.offload_kqv,
+                        cparams.kv_unified,
+                        cparams.n_ctx_seq,
+                        cparams.n_seq_max,
+                        hparams.n_swa,
+                        LLAMA_SWA_TYPE_STANDARD,
+                        nullptr,
+                        nullptr,
+                        nullptr,
+                        nullptr);
+            } break;
         case LLM_ARCH_HY_V4:
             {
                 if (hparams.indexer_top_k == 0) {
@@ -3789,6 +3812,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_DOTS3NOTE:
         case LLM_ARCH_NANBEIGE:
         case LLM_ARCH_POCKETTTS:
+        case LLM_ARCH_MOSHI:
         case LLM_ARCH_ZONOS3:
         case LLM_ARCH_LIMITE:
         case LLM_ARCH_TINY_VOCA:
diff --git a/src/models/models.h b/src/models/models.h
index 8d2a58827..0f49b8ae0 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -770,6 +770,18 @@ struct llama_model_pockettts : public llama_model_base {
     std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
 };

+struct llama_model_moshi : public llama_model_base {
+    llama_model_moshi(const struct llama_model_params & params) : llama_model_base(params) {}
+    void load_arch_hparams(llama_model_loader & ml) override;
+    void load_arch_tensors(llama_model_loader & ml) override;
+
+    struct graph : public llm_graph_context {
+        graph(const llm_graph_params & params);
+    };
+
+    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
 struct llama_model_kokoro : public llama_model_base {
     llama_model_kokoro(const struct llama_model_params & params) : llama_model_base(params) {}
     void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/src/models/moshi.cpp b/src/models/moshi.cpp
new file mode 100644
index 000000000..5cb63cd7e
--- /dev/null
+++ b/src/models/moshi.cpp
@@ -0,0 +1,132 @@
+#include "models.h"
+
+// backbone of the Moshi family (e.g. Hibiki): one position per audio frame, the input rows (text + audio codes)
+// are built by mtmd, the head predicts the next text token. Every layer attends to the same sliding window, see create_memory().
+
+void llama_model_moshi::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+    // the window of every layer, it is applied by the KV cache, see create_memory()
+    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+
+    switch (hparams.n_layer()) {
+        case 16: type = LLM_TYPE_1B; break; // Hibiki-M
+        case 24: type = LLM_TYPE_2B; break; // Hibiki
+        default: type = LLM_TYPE_UNKNOWN;
+    }
+}
+
+void llama_model_moshi::load_arch_tensors(llama_model_loader & ml) {
+    LLAMA_LOAD_LOCALS;
+
+    // the last vocab entries are input-only (e.g. the initial text token), the head has no logits for them
+    // a model built from metadata only has no tensor to read it from
+    const ggml_tensor * meta = ml.get_tensor_meta(tn(LLM_TENSOR_OUTPUT, "weight").str().c_str());
+    const int64_t n_vocab_out = meta ? meta->ne[1] : n_vocab;
+    if (n_vocab_out < 1 || n_vocab_out > n_vocab) {
+        throw std::runtime_error(format("tensor 'output.weight' has invalid head size %lld (vocab size %lld)", (long long) n_vocab_out, (long long) n_vocab));
+    }
+
+    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab_out}, 0);
+
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+
+        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+        create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
+
+        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd, n_ff}, 0);
+        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
+    }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_moshi::build_arch_graph(const llm_graph_params & params) const {
+    return std::make_unique<graph>(params);
+}
+
+llama_model_moshi::graph::graph(const llm_graph_params & params) : llm_graph_context(params) {
+    const int64_t n_embd_head = hparams.n_embd_head_v();
+
+    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+    GGML_ASSERT(n_embd_head == n_rot);
+
+    ggml_tensor * cur;
+    ggml_tensor * inpL = build_inp_embd(model.tok_embd);
+
+    ggml_tensor * inp_pos = build_inp_pos();
+
+    auto * inp_attn = build_attn_inp_kv();
+
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    for (int il = 0; il < n_layer; ++il) {
+        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_norm", il);
+
+        {
+            auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head, n_head_kv, il);
+
+            Qcur = build_rope(Qcur, inp_pos, il);
+            Kcur = build_rope(Kcur, inp_pos, il);
+
+            cb(Qcur, "Qcur", il);
+            cb(Kcur, "Kcur", il);
+            cb(Vcur, "Vcur", il);
+
+            cur = build_attn(inp_attn,
+                    model.layers[il].wo, NULL, model.layers[il].wo_s,
+                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_embd_head)), il);
+        }
+
+        if (il == n_layer - 1 && inp_out_ids) {
+            cur  = ggml_get_rows(ctx0,  cur, inp_out_ids);
+            inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
+        }
+
+        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);
+        cb(ffn_inp, "ffn_inp", il);
+
+        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "ffn_norm", il);
+
+        cur = build_ffn(cur,
+                model.layers[il].ffn_up,   NULL, NULL,
+                model.layers[il].ffn_gate, NULL, NULL,
+                model.layers[il].ffn_down, NULL, NULL,
+                NULL,
+                LLM_FFN_SILU, LLM_FFN_PAR, il);
+        cb(cur, "ffn_out", il);
+
+        cur = ggml_add(ctx0, cur, ffn_inp);
+
+        cur = build_cvec(cur, il);
+        cb(cur, "l_out", il);
+
+        inpL = cur;
+    }
+
+    cur = build_norm(inpL, model.output_norm, NULL, LLM_NORM_RMS, -1);
+
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    cur = build_lora_mm(model.output, cur, model.output_s);
+
+    // pad the logits of the input-only tokens with -inf
+    const int64_t pad = model.tok_embd->ne[1] - model.output->ne[1];
+    if (pad > 0) {
+        ggml_tensor * neg_inf = ggml_scale_bias(ctx0, ggml_view_1d(ctx0, model.output_norm, 1, 0), 0.0f, -INFINITY);
+        neg_inf = ggml_repeat_4d(ctx0, neg_inf, pad, cur->ne[1], 1, 1);
+        cur = ggml_concat(ctx0, cur, neg_inf, 0);
+    }
+
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+}
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index b939d6683..9bd4b82bf 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -253,6 +253,7 @@ mistral4            ,0   ,any     ,ADD+ADD                     ,      1
 mistral4            ,0   ,any     ,MUL+ADD                     ,      2
 mistral4            ,0   ,any     ,RMS_NORM+MUL                ,      9
 mistral4            ,0   ,any     ,UNARY+ADD+ARGSORT+GET_ROWS  ,      1
+moshi               ,0   ,any     ,RMS_NORM+MUL                ,      5
 mpt                 ,0   ,any     ,NORM+MUL+ADD                ,      5
 muse-glimmer        ,0   ,any     ,RMS_NORM+MUL                ,     10
 muse-glimmer        ,0   ,any     ,RMS_NORM+MUL+ADD            ,      3
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 9c37d60ed..cd9810e6d 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -96,6 +96,7 @@ add_library(mtmd
             models/pockettts-seanet.cpp
             models/pockettts-spkenc.cpp
             models/pockettts-gen.cpp
+            models/moshi.cpp
             models/soprano.cpp
             models/kokoro.cpp
             models/tiny-voca.cpp
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index fb75270ad..278fcdb27 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -433,7 +433,11 @@ struct mtmd_graph_qwen3tts_gen : mtmd_graph {

     // MTMD_SUBCTX_GEN_PROCESS_TOKENIZE: reference clip -> codes and their talker input rows
     ggml_cgraph * build_ref_encoder();
-    ggml_tensor * nearest_code(ggml_tensor * x, ggml_tensor * codebook) const;
+    static ggml_tensor * nearest_code(ggml_context * ctx0, ggml_tensor * x, ggml_tensor * codebook);
+    // split RVQ (mimi): the first codebook on x_first, the residual ones on x_rest, cb_rest is stacked on dim 2
+    // x_first, x_rest: [dim, T] -> one [T] code tensor per codebook; also used by the moshi encoder
+    static std::vector<ggml_tensor *> rvq_encode(ggml_context * ctx0, ggml_tensor * x_first, ggml_tensor * cb_first,
+                                                 ggml_tensor * x_rest, ggml_tensor * cb_rest);

     // reference encoder transformer, see encoder_config in speech_tokenizer/config.json
     static constexpr int   ref_enc_n_head     = 8;
@@ -529,6 +533,8 @@ struct mtmd_graph_qwen3tts_gen : mtmd_graph {
 // pocket-tts: SEANet convolution stack, shared by the voice encoder and the mimi decoder.
 // stateless unless state_in is populated: convs then pad instead of carrying left-context.
 //
+struct c2w_state_slot;
+
 struct mtmd_graph_pockettts_seanet : mtmd_graph {
     mtmd_graph_pockettts_seanet(const mtmd_graph & parent) : mtmd_graph(parent) {}
     ggml_cgraph * build() override { GGML_ABORT("call encode()/decode() instead"); }
@@ -542,6 +548,10 @@ struct mtmd_graph_pockettts_seanet : mtmd_graph {
     std::map<std::string, ggml_tensor *> state_in;
     mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;

+    // the "state_in_<slot>" inputs of the slots, then the "state_out_<slot>" outputs written by the graph
+    void state_inputs(const std::vector<c2w_state_slot> & slots);
+    void state_outputs(ggml_cgraph * graph) const;
+
     ggml_tensor * conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation,
                          bool pad_replicate = false, const std::string & state_name = "") const;
     ggml_tensor * conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride,
@@ -549,10 +559,15 @@ struct mtmd_graph_pockettts_seanet : mtmd_graph {
     ggml_tensor * res_unit(ggml_tensor * x, const mtmd_seanet::stage & stage, int dilation,
                            const std::string & state_prefix = "") const;

-    // x: [T, C] -> [T / hop, dim]
+    // x: [T, C] -> [T / hop, dim], streams when state_in is populated
     ggml_tensor * encode(ggml_tensor * x) const;
     // x: [T, dim] -> [T * hop, 1], streams when state_in is populated
     ggml_tensor * decode(ggml_tensor * x) const;
+
+    // mimi transformer, always streams (see list_pockettts_state_slots): x: [dim, n_pos] -> [dim, n_pos]
+    ggml_tensor * transformer(ggml_tensor * x, const std::vector<mtmd_layer> & layers, int n_head, float eps) const;
+    // mimi decoder: upsample, transformer, SEANet decoder; x: [dim, n_frames] -> [n_samples, 1]
+    ggml_tensor * mimi_decode(ggml_tensor * x, int n_head, float eps) const;
 };

 // mimi encoder + speaker_proj: reference waveform -> voice conditioning rows
@@ -594,6 +609,43 @@ struct mtmd_graph_pockettts_gen : mtmd_graph {
     ggml_tensor * flow_forward(ggml_tensor * cond, ggml_tensor * x, float s, float t) const;
 };

+//
+// moshi family (e.g. Hibiki): streaming mimi encoder + RVQ, rows of the input audio codes
+// output: [n_embd, n_q * (n_frames + 1)], the embedding of each code of each frame, then of the initial codes
+//
+struct mtmd_graph_moshi : mtmd_graph {
+    mtmd_graph_moshi(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
+    ggml_cgraph * build() override;
+
+    static constexpr int n_samples_per_frame = 1920; // 80 ms at 24 kHz
+};
+
+//
+// moshi generation:
+// GEN_CODE = depformer, the codes of one frame + their backbone embedding; with all codes known, only the embedding
+// GEN_WAV  = RVQ decode + mimi decoder
+//
+struct mtmd_graph_moshi_gen : mtmd_graph {
+    mtmd_graph_moshi_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, bool fb_only,
+                         int n_frames, int top_k, float temp)
+        : mtmd_graph(ctx, img), gen_process(gen_process), fb_only(fb_only), n_frames(n_frames), top_k(top_k), temp(temp) {}
+    ggml_cgraph * build() override;
+
+    static constexpr int   mimi_n_layer = 8;
+    static constexpr int   mimi_n_head  = 8;
+    static constexpr float mimi_eps     = 1e-5f;
+    static constexpr int   n_q_max      = 32;
+
+    mtmd_subctx_gen_process gen_process;
+    bool  fb_only;
+    int   n_frames;
+    int   top_k;
+    float temp;
+
+    ggml_tensor * dep_step(ggml_tensor * x, int64_t iw, std::vector<ggml_tensor *> & k_all, std::vector<ggml_tensor *> & v_all);
+    ggml_tensor * feedback(const std::vector<ggml_tensor *> & codes);
+};
+
 // one persisted state buffer used by code2wav, see qwen3tts-gen.cpp
 struct c2w_state_slot {
     std::string name;
@@ -609,6 +661,9 @@ std::vector<c2w_state_slot> list_drama3_state_slots(const mtmd_hparams & hparams
 // same, for the streaming mimi decoder (pocket-tts GEN_WAV)
 std::vector<c2w_state_slot> list_pockettts_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);

+// streaming mimi encoder of moshi, "pos" leads
+std::vector<c2w_state_slot> list_moshi_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
+
 //
 // drama3 generation:
 // GEN_CODE = one codebook tower row, each layer attends over the whole sequence (KV in gen state)
diff --git a/tools/mtmd/models/moshi.cpp b/tools/mtmd/models/moshi.cpp
new file mode 100644
index 000000000..ff99f10a0
--- /dev/null
+++ b/tools/mtmd/models/moshi.cpp
@@ -0,0 +1,265 @@
+#include "models.h"
+
+// Moshi family (e.g. Hibiki), see conversion/moshi.py
+//
+// encoder (audio stream): streaming mimi encoder -> RVQ -> backbone embeddings of the input codes
+// GEN_CODE: depformer, the codes of one frame from the backbone hidden state and the text token, then their embedding
+// GEN_WAV : RVQ decode -> streaming mimi decoder
+
+// 2D slice i of a tensor stacked on dim 2
+static ggml_tensor * moshi_slice(ggml_context * ctx0, ggml_tensor * w, int64_t i) {
+    return ggml_view_2d(ctx0, w, w->ne[0], w->ne[1], w->nb[1], (size_t) i * w->nb[2]);
+}
+
+std::vector<c2w_state_slot> list_moshi_state_slots(const mtmd_hparams & hparams, const mtmd_model & model) {
+    std::vector<c2w_state_slot> slots;
+    const auto & seanet = model.seanet;
+
+    slots.push_back({"pos", 1, 1});
+
+    const int64_t n_embd_a = model.layers[0].q_w->ne[1];
+    const int64_t prefix   = hparams.mimi_tfm_context - 1;
+    for (size_t il = 0; il < model.layers.size(); il++) {
+        slots.push_back({"tfm_k_" + std::to_string(il), n_embd_a, prefix});
+        slots.push_back({"tfm_v_" + std::to_string(il), n_embd_a, prefix});
+    }
+
+    // left context of the causal convs, [kernel - stride, in channels]
+    slots.push_back({"enc_in", seanet.conv_in_w->ne[0] - 1, seanet.conv_in_w->ne[1]});
+    for (int i = 0; i < hparams.seanet_n_stage; i++) {
+        const auto & stage = seanet.stages[i];
+        slots.push_back({"enc_res_" + std::to_string(i), stage.res_conv1_w->ne[0] - 1, stage.res_conv1_w->ne[1]});
+        slots.push_back({"enc_down_" + std::to_string(i), stage.scale_conv_w->ne[0] - hparams.seanet_ratios[i], stage.scale_conv_w->ne[1]});
+    }
+    slots.push_back({"enc_out", seanet.conv_out_w->ne[0] - 1, seanet.conv_out_w->ne[1]});
+    slots.push_back({"down", model.downsample_w->ne[0] - hparams.mimi_downsample, model.downsample_w->ne[1]});
+
+    return slots;
+}
+
+// waveform of whole frames -> [n_embd, n_q, n_frames + 1]: the embedding of each input code, then of the initial codes
+ggml_cgraph * mtmd_graph_moshi::build() {
+    // the waveform comes in as a single-row "mel", [n_samples, 1]
+    ggml_tensor * inp_raw = build_inp_raw(1);
+    ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
+    GGML_ASSERT(cur->ne[0] % n_samples_per_frame == 0);
+
+    // the graph always streams, a zero state encodes a clip from its start
+    mtmd_graph_pockettts_seanet seanet(*this);
+    seanet.state_inputs(list_moshi_state_slots(hparams, model));
+    ggml_tensor * pos_in = seanet.state_in.at("pos");
+
+    cur = seanet.encode(cur);
+    cb(cur, "mimi_enc", -1);
+
+    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+    cur = seanet.transformer(cur, model.layers, hparams.n_head, hparams.eps);
+    cb(cur, "mimi_enc_tfm", -1);
+
+    // the downsample conv repeats the first frame on its left at the stream start, the state is zero then
+    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+    {
+        const int64_t n_left = seanet.state_in.at("down")->ne[0];
+        ggml_tensor * first    = ggml_repeat_4d(ctx0, ggml_view_2d(ctx0, cur, 1, cur->ne[1], cur->nb[1], 0), n_left, cur->ne[1], 1, 1);
+        ggml_tensor * is_first = ggml_step(ctx0, ggml_scale_bias(ctx0, pos_in, -1.0f, 0.5f));
+        seanet.state_in["down"] = ggml_add(ctx0, seanet.state_in.at("down"), ggml_mul(ctx0, first, is_first));
+    }
+    cur = seanet.conv1d(cur, model.downsample_w, nullptr, hparams.mimi_downsample, 1, false, "down");
+    cb(cur, "mimi_downsample", -1);
+
+    seanet.state_outputs(gf);
+
+    // split RVQ: the semantic codebook, then the residual acoustic ones, all from the same latent
+    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [512, n_frames]
+    const auto &  c2w      = model.c2w;
+    const int64_t n_frames = cur->ne[1];
+    const int64_t n_q      = model.mm_a_code_embd->ne[2];
+    const int64_t n_embd   = model.mm_a_code_embd->ne[0];
+
+    const std::vector<ggml_tensor *> codes = mtmd_graph_qwen3tts_gen::rvq_encode(ctx0,
+        build_mm(c2w.quant_first_in_w, cur), c2w.quant_first_cb_w, build_mm(c2w.quant_rest_in_w, cur), c2w.quant_rest_cb_w);
+    GGML_ASSERT((int64_t) codes.size() == n_q);
+
+    // [n_embd, n_frames, n_q] -> [n_embd, n_q, n_frames]
+    ggml_tensor * embd = nullptr;
+    for (int64_t k = 0; k < n_q; k++) {
+        cb(codes[k], "in_code", (int) k);
+        ggml_tensor * e = ggml_get_rows(ctx0, moshi_slice(ctx0, model.mm_a_code_embd, k), codes[k]);
+        e = ggml_reshape_3d(ctx0, e, n_embd, n_frames, 1);
+        embd = embd ? ggml_concat(ctx0, embd, e, 2) : e;
+    }
+    embd = ggml_cont(ctx0, ggml_permute(ctx0, embd, 0, 2, 1, 3));
+
+    // the initial code is the last entry of every table
+    const float   code_initial = (float) (model.mm_a_code_embd->ne[1] - 1);
+    ggml_tensor * idx_initial  = ggml_cast(ctx0, ggml_arange(ctx0, code_initial, code_initial + 1.0f, 1.0f), GGML_TYPE_I32);
+    ggml_tensor * init = nullptr;
+    for (int64_t k = 0; k < n_q; k++) {
+        ggml_tensor * e = ggml_get_rows(ctx0, moshi_slice(ctx0, model.mm_a_code_embd, k), idx_initial);
+        init = init ? ggml_concat(ctx0, init, e, 1) : e;
+    }
+    embd = ggml_concat(ctx0, embd, ggml_reshape_3d(ctx0, init, n_embd, n_q, 1), 2);
+
+    embd = ggml_reshape_2d(ctx0, embd, n_embd, n_q * (n_frames + 1));
+    cb(embd, "moshi_in_embd", -1);
+    ggml_build_forward_expand(gf, embd);
+
+    return gf;
+}
+
+// one depformer step: input row x [n_dep, 1], the K/V of the steps so far grow by one row per layer
+ggml_tensor * mtmd_graph_moshi_gen::dep_step(ggml_tensor * x, int64_t iw, std::vector<ggml_tensor *> & k_all, std::vector<ggml_tensor *> & v_all) {
+    const int64_t n_dep  = x->ne[0];
+    const int64_t d_head = n_dep / n_head;
+
+    for (size_t il = 0; il < model.layers.size(); il++) {
+        const auto & layer = model.layers[il];
+        ggml_tensor * inp = x;
+
+        ggml_tensor * cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, x, eps), layer.ln_1_w);
+        ggml_tensor * q = ggml_mul_mat(ctx0, moshi_slice(ctx0, layer.q_w, iw), cur);
+        ggml_tensor * k = ggml_mul_mat(ctx0, moshi_slice(ctx0, layer.k_w, iw), cur);
+        ggml_tensor * v = ggml_mul_mat(ctx0, moshi_slice(ctx0, layer.v_w, iw), cur);
+
+        // no positional embedding, causal over the steps of this frame
+        k_all[il] = k_all[il] ? ggml_concat(ctx0, k_all[il], k, 1) : k;
+        v_all[il] = v_all[il] ? ggml_concat(ctx0, v_all[il], v, 1) : v;
+        const int64_t n_kv = k_all[il]->ne[1];
+
+        cur = build_attn(nullptr, nullptr,
+                         ggml_reshape_4d(ctx0, q,         d_head, n_head, 1,    1),
+                         ggml_reshape_4d(ctx0, k_all[il], d_head, n_head, n_kv, 1),
+                         ggml_reshape_4d(ctx0, v_all[il], d_head, n_head, n_kv, 1),
+                         nullptr, 1.0f / sqrtf((float) d_head), (int) il);
+        x = ggml_add(ctx0, inp, ggml_mul_mat(ctx0, moshi_slice(ctx0, layer.o_w, iw), cur));
+
+        cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, x, eps), layer.ln_2_w);
+        cur = ggml_swiglu_split(ctx0, ggml_mul_mat(ctx0, moshi_slice(ctx0, layer.ff_gate_w, iw), cur),
+                                      ggml_mul_mat(ctx0, moshi_slice(ctx0, layer.ff_up_w,   iw), cur));
+        x = ggml_add(ctx0, x, ggml_mul_mat(ctx0, moshi_slice(ctx0, layer.ff_down_w, iw), cur));
+    }
+    return x;
+}
+
+// backbone input of the next frame: sum of the embeddings of the generated codes
+ggml_tensor * mtmd_graph_moshi_gen::feedback(const std::vector<ggml_tensor *> & codes) {
+    ggml_tensor * fb = nullptr;
+    for (size_t k = 0; k < codes.size(); k++) {
+        ggml_tensor * e = ggml_get_rows(ctx0, moshi_slice(ctx0, model.gen_code_out_embd_w, (int64_t) k), codes[k]);
+        fb = fb ? ggml_add(ctx0, fb, e) : e;
+    }
+    return fb;
+}
+
+ggml_cgraph * mtmd_graph_moshi_gen::build() {
+    const int64_t n_q = model.gen_code_head_w->ne[2];
+
+    if (gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
+        // codebook-major: row k holds codebook k of every frame
+        ggml_tensor * inp_codes = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, n_frames, n_q);
+        ggml_set_name(inp_codes, "inp_codes");
+        ggml_set_input(inp_codes);
+
+        auto codes_of = [&](int64_t k) {
+            return ggml_view_1d(ctx0, inp_codes, n_frames, (size_t) k * inp_codes->nb[1]);
+        };
+
+        // RVQ decode: output_proj(sum of the codebook entries) of the semantic and acoustic parts
+        const auto & c2w = model.c2w;
+        ggml_tensor * rest = nullptr;
+        for (int64_t k = 1; k < n_q; k++) {
+            ggml_tensor * e = ggml_get_rows(ctx0, moshi_slice(ctx0, c2w.quant_rest_cb_w, k - 1), codes_of(k));
+            rest = rest ? ggml_add(ctx0, rest, e) : e;
+        }
+        ggml_tensor * cur = build_mm(c2w.quant_first_out_w, ggml_get_rows(ctx0, c2w.quant_first_cb_w, codes_of(0)));
+        cur = ggml_add(ctx0, cur, build_mm(c2w.quant_rest_out_w, rest));
+        cb(cur, "rvq_decode", -1);
+
+        mtmd_graph_pockettts_seanet seanet(*this);
+        seanet.state_inputs(list_pockettts_state_slots(hparams, model));
+
+        cur = seanet.mimi_decode(cur, mimi_n_head, mimi_eps);
+
+        seanet.state_outputs(gf);
+
+        cur = ggml_reshape_1d(ctx0, cur, cur->ne[0]);
+        ggml_set_name(cur, "out_audio");
+        ggml_set_output(cur);
+        ggml_build_forward_expand(gf, cur);
+        return gf;
+    }
+
+    // GEN_CODE with all codes known: only their embedding
+    if (fb_only) {
+        ggml_tensor * inp_codes = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_q);
+        ggml_set_name(inp_codes, "inp_codes");
+        ggml_set_input(inp_codes);
+
+        std::vector<ggml_tensor *> codes;
+        for (int64_t k = 0; k < n_q; k++) {
+            codes.push_back(ggml_view_1d(ctx0, inp_codes, 1, (size_t) k * sizeof(int32_t)));
+        }
+        ggml_tensor * fb = feedback(codes);
+        cb(fb, "gen_audio_out", -1);
+        ggml_build_forward_expand(gf, fb);
+        return gf;
+    }
+
+    // the backbone hidden state arrives as the single batch entry
+    ggml_tensor * h = ggml_reshape_2d(ctx0, build_inp_raw(1), n_mmproj_embd, 1);
+
+    ggml_tensor * inp_code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); // the sampled text token
+    ggml_set_name(inp_code0, "inp_code0");
+    ggml_set_input(inp_code0);
+
+    ggml_tensor * inp_rand = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_q);
+    ggml_set_name(inp_rand, "inp_rand");
+    ggml_set_input(inp_rand);
+
+    mtmd_graph_qwen3tts_gen::code_gen cg(*this, top_k, 1.0f);
+
+    // steps past the last weight set reuse it
+    const int64_t n_w = model.gen_code_proj_in_w->ne[2];
+
+    std::vector<ggml_tensor *> k_all(model.layers.size(), nullptr);
+    std::vector<ggml_tensor *> v_all(model.layers.size(), nullptr);
+    std::vector<ggml_tensor *> codes;
+    ggml_tensor * out_codes = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, 1, n_q);
+
+    for (int64_t k = 0; k < n_q; k++) {
+        const int64_t iw = std::min(k, n_w - 1);
+
+        // the previous token: the text token, then the codes of this frame
+        ggml_tensor * prev = k == 0 ? ggml_get_rows(ctx0, model.gen_code_tok_embd_w, inp_code0)
+                                    : ggml_get_rows(ctx0, moshi_slice(ctx0, model.gen_code_embd_w, k - 1), codes.back());
+        ggml_tensor * x = ggml_add(ctx0, ggml_mul_mat(ctx0, moshi_slice(ctx0, model.gen_code_proj_in_w, iw), h), prev);
+
+        x = dep_step(x, iw, k_all, v_all);
+
+        ggml_tensor * logits = ggml_mul_mat(ctx0, moshi_slice(ctx0, model.gen_code_head_w, k), x);
+        cb(logits, "dep_logits", (int) k);
+
+        ggml_tensor * code;
+        if (temp <= 0.0f) {
+            code = ggml_argmax(ctx0, logits);
+        } else {
+            if (temp != 1.0f) {
+                logits = ggml_scale(ctx0, logits, 1.0f / temp);
+            }
+            code = cg.do_sampling(logits, ggml_view_1d(ctx0, inp_rand, 1, (size_t) k * sizeof(float)));
+        }
+        codes.push_back(code);
+        out_codes = cg.cache_set(out_codes, (int) k, code);
+    }
+
+    out_codes = ggml_cont(ctx0, out_codes);
+    ggml_set_name(out_codes, "out_codes");
+    ggml_set_output(out_codes);
+    ggml_build_forward_expand(gf, out_codes);
+
+    ggml_tensor * fb = feedback(codes);
+    cb(fb, "gen_audio_out", -1);
+    ggml_build_forward_expand(gf, fb);
+
+    return gf;
+}
diff --git a/tools/mtmd/models/pockettts-gen.cpp b/tools/mtmd/models/pockettts-gen.cpp
index c5f340656..db38b8b69 100644
--- a/tools/mtmd/models/pockettts-gen.cpp
+++ b/tools/mtmd/models/pockettts-gen.cpp
@@ -112,7 +112,7 @@ std::vector<c2w_state_slot> list_pockettts_state_slots(const mtmd_hparams & hpar
     GGML_ASSERT((int) hparams.seanet_ratios.size() >= hparams.seanet_n_stage);
     GGML_ASSERT(hparams.mimi_tfm_context > 1 && hparams.mimi_downsample > 0);

-    slots.push_back({"tfm_pos", 1, 1});
+    slots.push_back({"pos", 1, 1});

     const int64_t n_embd_a = model.gen_tfm_layers[0].q_w->ne[1];
     const int64_t prefix   = hparams.mimi_tfm_context - 1;
@@ -189,99 +189,11 @@ ggml_cgraph * mtmd_graph_pockettts_gen::build() {
     cb(cur, "quant_out", -1);

     mtmd_graph_pockettts_seanet seanet(*this);
-    for (const auto & slot : list_pockettts_state_slots(hparams, model)) {
-        ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
-        ggml_set_name(t, ("state_in_" + slot.name).c_str());
-        ggml_set_input(t);
-        seanet.state_in[slot.name] = t;
-    }
-
-    // model frame rate -> encoder frame rate, depthwise transposed conv
-    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
-    cur = seanet.conv_transpose1d(cur, model.gen_upsample_w, nullptr, hparams.mimi_downsample, "up");
-    cb(cur, "mimi_upsample", -1);
-
-    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
-
-    // positions continue across calls, the counter lives in the state
-    const int64_t n_pos  = cur->ne[1];
-    const int64_t prefix = hparams.mimi_tfm_context - 1;
-    const int64_t n_kv   = prefix + n_pos;
-
-    ggml_tensor * base    = ggml_reshape_1d(ctx0, seanet.state_in.at("tfm_pos"), 1);
-    ggml_tensor * inp_pos = ggml_cast(ctx0, ggml_add(ctx0, ggml_arange(ctx0, 0.0f, (float) n_pos, 1.0f), base),
-                                      GGML_TYPE_I32);
-    seanet.state_out.push_back({"tfm_pos", ggml_scale_bias(ctx0, seanet.state_in.at("tfm_pos"), 1.0f, (float) n_pos)});
-
-    // banded causal mask over [cached prefix | this chunk]
-    // the last factor masks out cache rows that hold no real frame yet
-    ggml_tensor * pos_k = ggml_reshape_2d(ctx0, ggml_arange(ctx0, 0.0f, (float) n_kv, 1.0f), n_kv, 1);
-    ggml_tensor * pos_q = ggml_reshape_2d(ctx0, ggml_arange(ctx0, (float) prefix, (float) (prefix + n_pos), 1.0f), 1, n_pos);
-    ggml_tensor * diff  = ggml_sub(ctx0, ggml_repeat_4d(ctx0, pos_q, n_kv, n_pos, 1, 1), pos_k);
-
-    ggml_tensor * keep = ggml_mul(ctx0,
-        ggml_step(ctx0, ggml_scale_bias(ctx0, diff, 1.0f, 0.5f)),                                     // delta >= 0
-        ggml_step(ctx0, ggml_scale_bias(ctx0, diff, -1.0f, (float) hparams.mimi_tfm_context - 0.5f))); // delta < context
-    keep = ggml_mul(ctx0, keep,
-        ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base), 1.0f, 0.5f - (float) prefix)));
-    ggml_tensor * kq_mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), n_kv, n_pos, 1, 1);
-    if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) {
-        kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16);
-    }
+    seanet.state_inputs(list_pockettts_state_slots(hparams, model));

-    for (int il = 0; il < n_layer; il++) {
-        const auto & layer = model.gen_tfm_layers[il];
-        ggml_tensor * inp = cur;
-
-        cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
-
-        ggml_tensor * Qcur = build_mm(layer.q_w, cur);
-        ggml_tensor * Kcur = build_mm(layer.k_w, cur);
-        ggml_tensor * Vcur = build_mm(layer.v_w, cur);
-
-        Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
-        Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
-
-        Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
-                             hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
-        Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
-                             hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
-
-        // prepend the cached window, then keep this chunk's tail for the next call
-        const std::string k_name = "tfm_k_" + std::to_string(il);
-        const std::string v_name = "tfm_v_" + std::to_string(il);
-        ggml_tensor * k_full = ggml_concat(ctx0, seanet.state_in.at(k_name),
-                                           ggml_reshape_2d(ctx0, Kcur, d_head * n_head, n_pos), 1);
-        ggml_tensor * v_full = ggml_concat(ctx0, seanet.state_in.at(v_name), Vcur, 1);
-        seanet.state_out.push_back({k_name, ggml_cont(ctx0, ggml_view_2d(ctx0, k_full, k_full->ne[0], prefix,
-                                                                         k_full->nb[1], (size_t) n_pos * k_full->nb[1]))});
-        seanet.state_out.push_back({v_name, ggml_cont(ctx0, ggml_view_2d(ctx0, v_full, v_full->ne[0], prefix,
-                                                                         v_full->nb[1], (size_t) n_pos * v_full->nb[1]))});
-
-        ggml_tensor * q_cur = ggml_reshape_4d(ctx0, Qcur,   d_head, n_head, n_pos, 1);
-        ggml_tensor * k_cur = ggml_reshape_4d(ctx0, k_full, d_head, n_head, n_kv,  1);
-        ggml_tensor * v_cur = ggml_reshape_4d(ctx0, v_full, d_head, n_head, n_kv,  1);
-
-        cur = build_attn(layer.o_w, nullptr, q_cur, k_cur, v_cur, kq_mask, kq_scale, il);
-        cur = ggml_mul(ctx0, cur, layer.ls_1_w);
-        cur = ggml_add(ctx0, cur, inp);
-
-        inp = cur;
-        cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
-        cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, FFN_GELU, il);
-        cur = ggml_mul(ctx0, cur, layer.ls_2_w);
-        cur = ggml_add(ctx0, cur, inp);
-    }
-    cb(cur, "mimi_dec_tfm", -1);
-
-    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
-    cur = seanet.decode(cur);
+    cur = seanet.mimi_decode(cur, n_head, eps);

-    for (const auto & s : seanet.state_out) {
-        ggml_set_name(s.second, ("state_out_" + s.first).c_str());
-        ggml_set_output(s.second);
-        ggml_build_forward_expand(gf, s.second);
-    }
+    seanet.state_outputs(gf);

     // [n_samples, 1] -> [n_samples], clamped like the reference output
     cur = ggml_reshape_1d(ctx0, cur, cur->ne[0]);
diff --git a/tools/mtmd/models/pockettts-seanet.cpp b/tools/mtmd/models/pockettts-seanet.cpp
index f1073c844..e0a97bdc6 100644
--- a/tools/mtmd/models/pockettts-seanet.cpp
+++ b/tools/mtmd/models/pockettts-seanet.cpp
@@ -125,22 +125,27 @@ ggml_tensor * mtmd_graph_pockettts_seanet::res_unit(ggml_tensor * x, const mtmd_

 ggml_tensor * mtmd_graph_pockettts_seanet::encode(ggml_tensor * x) const {
     const auto & seanet = model.seanet;
+    const bool   stream = !state_in.empty();

-    ggml_tensor * cur = conv1d(x, seanet.conv_in_w, seanet.conv_in_b, 1, 1);
+    ggml_tensor * cur = conv1d(x, seanet.conv_in_w, seanet.conv_in_b, 1, 1, false,
+                               stream ? "enc_in" : "");
     cb(cur, "seanet_enc_in", -1);

     for (int i = 0; i < hparams.seanet_n_stage; i++) {
         const auto & stage  = seanet.stages[i];
         const int    stride = hparams.seanet_ratios[i];
+        const std::string id = std::to_string(i);

-        cur = res_unit(cur, stage, 1);
+        cur = res_unit(cur, stage, 1, stream ? "enc_res_" + id : "");
         cur = ggml_elu(ctx0, cur);
-        cur = conv1d(cur, stage.scale_conv_w, stage.scale_conv_b, stride, 1);
+        cur = conv1d(cur, stage.scale_conv_w, stage.scale_conv_b, stride, 1, false,
+                     stream ? "enc_down_" + id : "");
         cb(cur, "seanet_enc_stage", i);
     }

     cur = ggml_elu(ctx0, cur);
-    cur = conv1d(cur, seanet.conv_out_w, seanet.conv_out_b, 1, 1);
+    cur = conv1d(cur, seanet.conv_out_w, seanet.conv_out_b, 1, 1, false,
+                 stream ? "enc_out" : "");
     cb(cur, "seanet_enc_out", -1);

     return cur;
@@ -174,3 +179,112 @@ ggml_tensor * mtmd_graph_pockettts_seanet::decode(ggml_tensor * x) const {

     return cur;
 }
+
+// pre-norm blocks with layer scale on both residual paths, see mimi_transformer.py
+// causal over a window of mimi_tfm_context frames, streams through the slots pos, tfm_k_<il>, tfm_v_<il>
+// x: [dim, n_pos] -> [dim, n_pos]
+ggml_tensor * mtmd_graph_pockettts_seanet::transformer(ggml_tensor * cur, const std::vector<mtmd_layer> & layers, int n_head, float eps) const {
+    const int64_t n_pos  = cur->ne[1];
+    const int64_t d_head = cur->ne[0] / n_head;
+    const int64_t prefix = hparams.mimi_tfm_context - 1;
+    const int64_t n_kv   = prefix + n_pos;
+    const float   scale  = 1.0f / sqrtf((float) d_head);
+
+    // positions continue across calls, the counter lives in the state
+    ggml_tensor * base    = ggml_reshape_1d(ctx0, state_in.at("pos"), 1);
+    ggml_tensor * inp_pos = ggml_cast(ctx0, ggml_add(ctx0, ggml_arange(ctx0, 0.0f, (float) n_pos, 1.0f), base),
+                                      GGML_TYPE_I32);
+    state_out.push_back({"pos", ggml_scale_bias(ctx0, state_in.at("pos"), 1.0f, (float) n_pos)});
+
+    // banded causal mask over [cached prefix | this chunk]
+    // the last factor masks out cache rows that hold no real frame yet
+    ggml_tensor * pos_k = ggml_reshape_2d(ctx0, ggml_arange(ctx0, 0.0f, (float) n_kv, 1.0f), n_kv, 1);
+    ggml_tensor * pos_q = ggml_reshape_2d(ctx0, ggml_arange(ctx0, (float) prefix, (float) (prefix + n_pos), 1.0f), 1, n_pos);
+    ggml_tensor * diff  = ggml_sub(ctx0, ggml_repeat_4d(ctx0, pos_q, n_kv, n_pos, 1, 1), pos_k);
+
+    ggml_tensor * keep = ggml_mul(ctx0,
+        ggml_step(ctx0, ggml_scale_bias(ctx0, diff, 1.0f, 0.5f)),                                     // delta >= 0
+        ggml_step(ctx0, ggml_scale_bias(ctx0, diff, -1.0f, (float) hparams.mimi_tfm_context - 0.5f))); // delta < context
+    keep = ggml_mul(ctx0, keep,
+        ggml_step(ctx0, ggml_scale_bias(ctx0, ggml_add(ctx0, pos_k, base), 1.0f, 0.5f - (float) prefix)));
+    ggml_tensor * kq_mask = ggml_reshape_4d(ctx0, ggml_log(ctx0, keep), n_kv, n_pos, 1, 1);
+    if (flash_attn_type == MTMD_FLASH_ATTN_TYPE_ENABLED) {
+        kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16);
+    }
+
+    for (size_t il = 0; il < layers.size(); il++) {
+        const auto & layer = layers[il];
+        ggml_tensor * inp = cur;
+
+        cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
+
+        ggml_tensor * Qcur = build_mm(layer.q_w, cur);
+        ggml_tensor * Kcur = build_mm(layer.k_w, cur);
+        ggml_tensor * Vcur = build_mm(layer.v_w, cur);
+
+        Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
+        Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
+
+        Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
+                             hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+        Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
+                             hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+
+        // prepend the cached window, then keep this chunk's tail for the next call
+        const std::string k_name = "tfm_k_" + std::to_string(il);
+        const std::string v_name = "tfm_v_" + std::to_string(il);
+        ggml_tensor * k_full = ggml_concat(ctx0, state_in.at(k_name),
+                                           ggml_reshape_2d(ctx0, Kcur, d_head * n_head, n_pos), 1);
+        ggml_tensor * v_full = ggml_concat(ctx0, state_in.at(v_name), Vcur, 1);
+        state_out.push_back({k_name, ggml_cont(ctx0, ggml_view_2d(ctx0, k_full, k_full->ne[0], prefix,
+                                                                  k_full->nb[1], (size_t) n_pos * k_full->nb[1]))});
+        state_out.push_back({v_name, ggml_cont(ctx0, ggml_view_2d(ctx0, v_full, v_full->ne[0], prefix,
+                                                                  v_full->nb[1], (size_t) n_pos * v_full->nb[1]))});
+
+        ggml_tensor * q_cur = ggml_reshape_4d(ctx0, Qcur,   d_head, n_head, n_pos, 1);
+        ggml_tensor * k_cur = ggml_reshape_4d(ctx0, k_full, d_head, n_head, n_kv,  1);
+        ggml_tensor * v_cur = ggml_reshape_4d(ctx0, v_full, d_head, n_head, n_kv,  1);
+
+        cur = build_attn(layer.o_w, nullptr, q_cur, k_cur, v_cur, kq_mask, scale, il);
+        cur = ggml_mul(ctx0, cur, layer.ls_1_w);
+        cur = ggml_add(ctx0, cur, inp);
+
+        inp = cur;
+        cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
+        cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, FFN_GELU, il);
+        cur = ggml_mul(ctx0, cur, layer.ls_2_w);
+        cur = ggml_add(ctx0, cur, inp);
+    }
+    return cur;
+}
+
+ggml_tensor * mtmd_graph_pockettts_seanet::mimi_decode(ggml_tensor * x, int n_head, float eps) const {
+    // model frame rate -> encoder frame rate, depthwise transposed conv
+    ggml_tensor * cur = ggml_cont(ctx0, ggml_transpose(ctx0, x));
+    cur = conv_transpose1d(cur, model.gen_upsample_w, nullptr, hparams.mimi_downsample, "up");
+    cb(cur, "mimi_upsample", -1);
+
+    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+    cur = transformer(cur, model.gen_tfm_layers, n_head, eps);
+    cb(cur, "mimi_dec_tfm", -1);
+
+    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+    return decode(cur);
+}
+
+void mtmd_graph_pockettts_seanet::state_inputs(const std::vector<c2w_state_slot> & slots) {
+    for (const auto & slot : slots) {
+        ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
+        ggml_set_name(t, ("state_in_" + slot.name).c_str());
+        ggml_set_input(t);
+        state_in[slot.name] = t;
+    }
+}
+
+void mtmd_graph_pockettts_seanet::state_outputs(ggml_cgraph * graph) const {
+    for (const auto & s : state_out) {
+        ggml_set_name(s.second, ("state_out_" + s.first).c_str());
+        ggml_set_output(s.second);
+        ggml_build_forward_expand(graph, s.second);
+    }
+}
diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp
index a453b09f5..8cee94407 100644
--- a/tools/mtmd/models/qwen3tts-gen.cpp
+++ b/tools/mtmd/models/qwen3tts-gen.cpp
@@ -665,7 +665,7 @@ std::vector<c2w_state_slot> list_c2w_state_slots(const mtmd_hparams & hparams, c
 }

 // x: [dim, T], codebook: [dim, n_entries] -> [T] index of the nearest entry
-ggml_tensor * mtmd_graph_qwen3tts_gen::nearest_code(ggml_tensor * x, ggml_tensor * codebook) const {
+ggml_tensor * mtmd_graph_qwen3tts_gen::nearest_code(ggml_context * ctx0, ggml_tensor * x, ggml_tensor * codebook) {
     // argmin |x - e|^2 = argmax 2 x.e - |e|^2
     ggml_tensor * dots  = ggml_mul_mat(ctx0, codebook, x);                // [n_entries, T]
     ggml_tensor * norms = ggml_sum_rows(ctx0, ggml_sqr(ctx0, codebook));  // [1, n_entries]
@@ -673,6 +673,24 @@ ggml_tensor * mtmd_graph_qwen3tts_gen::nearest_code(ggml_tensor * x, ggml_tensor
     return ggml_argmax(ctx0, ggml_sub(ctx0, ggml_scale(ctx0, dots, 2.0f), norms));
 }

+std::vector<ggml_tensor *> mtmd_graph_qwen3tts_gen::rvq_encode(ggml_context * ctx0, ggml_tensor * x_first, ggml_tensor * cb_first,
+                                                                ggml_tensor * x_rest, ggml_tensor * cb_rest) {
+    std::vector<ggml_tensor *> codes;
+    codes.push_back(nearest_code(ctx0, x_first, cb_first));
+
+    const int64_t n_rest = cb_rest->ne[2];
+    ggml_tensor * residual = x_rest;
+    for (int64_t i = 0; i < n_rest; i++) {
+        ggml_tensor * codebook = ggml_view_2d(ctx0, cb_rest, cb_rest->ne[0], cb_rest->ne[1], cb_rest->nb[1], (size_t) i * cb_rest->nb[2]);
+        ggml_tensor * code = nearest_code(ctx0, residual, codebook);
+        codes.push_back(code);
+        if (i + 1 < n_rest) {
+            residual = ggml_sub(ctx0, residual, ggml_get_rows(ctx0, codebook, code));
+        }
+    }
+    return codes;
+}
+
 // the mimi encoder of the speech tokenizer, then its RVQ: one semantic codebook and n_acoustic acoustic ones
 ggml_cgraph * mtmd_graph_qwen3tts_gen::build_ref_encoder() {
     // the waveform comes in as a single-row "mel", [n_samples, 1]
@@ -694,20 +712,9 @@ ggml_cgraph * mtmd_graph_qwen3tts_gen::build_ref_encoder() {
     cur = mimi.encode(seanet, cur, model.ref_enc_layers);
     cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [512, T]

-    std::vector<ggml_tensor *> codes;
-    codes.push_back(nearest_code(build_mm(model.ref_quant_first_in_w, cur), model.ref_quant_first_cb));
-
+    const std::vector<ggml_tensor *> codes = rvq_encode(ctx0, build_mm(model.ref_quant_first_in_w, cur), model.ref_quant_first_cb,
+                                                        build_mm(model.ref_quant_rest_in_w, cur), model.ref_quant_rest_cb);
     const int64_t n_acoustic = model.ref_quant_rest_cb->ne[2];
-    ggml_tensor * residual = build_mm(model.ref_quant_rest_in_w, cur);
-    for (int64_t i = 0; i < n_acoustic; i++) {
-        ggml_tensor * codebook = ggml_view_2d(ctx0, model.ref_quant_rest_cb, model.ref_quant_rest_cb->ne[0], model.ref_quant_rest_cb->ne[1],
-                                              model.ref_quant_rest_cb->nb[1], (size_t) i * model.ref_quant_rest_cb->nb[2]);
-        ggml_tensor * code = nearest_code(residual, codebook);
-        codes.push_back(code);
-        if (i + 1 < n_acoustic) {
-            residual = ggml_sub(ctx0, residual, ggml_get_rows(ctx0, codebook, code));
-        }
-    }

     // talker input rows of the reference frames: codebook 0 from the talker table, the rest from the code predictor's
     ggml_tensor * rows = ggml_get_rows(ctx0, model.gen_code_out_embd_w, codes[0]);
diff --git a/tools/mtmd/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp
index 4534527fc..8fc8d1b89 100644
--- a/tools/mtmd/mtmd-cli.cpp
+++ b/tools/mtmd/mtmd-cli.cpp
@@ -120,6 +120,10 @@ struct mtmd_cli_context {
         init_vision_context(params);

         mtmd_cap = mtmd_helper_get_capabilities(model, ctx_vision.get());
+        if (mtmd_cap.audio_duplex) {
+            LOG_ERR("Full duplex models only run in llama-server, see the /v1/realtime endpoint\n");
+            exit(1);
+        }
         // a realtime transcription model has no chat, it streams the audio below
         if (!mtmd_cap.chat && !mtmd_cap.audio_stream) {
             LOG_ERR("Model does not support chat mode\n");
diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp
index 81ae155ac..a22f41d37 100644
--- a/tools/mtmd/mtmd-graph.cpp
+++ b/tools/mtmd/mtmd-graph.cpp
@@ -1000,6 +1000,24 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
             {
                 builder = std::make_unique<mtmd_graph_chatterbox_spkenc>(ctx, img);
             } break;
+        case PROJECTOR_TYPE_MOSHI:
+            {
+                builder = std::make_unique<mtmd_graph_moshi>(ctx, img);
+            } break;
+        case PROJECTOR_TYPE_MOSHI_GEN:
+            {
+                const auto    gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
+                const int64_t n_q         = ctx->model.gen_code_head_w->ne[2];
+                const bool    has_codes   = params && params->codes;
+                if (has_codes) {
+                    GGML_ASSERT(params->codes->size() % (size_t) n_q == 0);
+                }
+                const int   n_frames = has_codes ? (int) (params->codes->size() / n_q) : 1;
+                const int   top_k    = params ? params->top_k : 250;
+                const float temp     = params ? params->temp  : 0.8f;
+                const bool  fb_only  = gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE && has_codes;
+                builder = std::make_unique<mtmd_graph_moshi_gen>(ctx, img, gen_process, fb_only, n_frames, top_k, temp);
+            } break;
         case PROJECTOR_TYPE_CHATTERBOX:
             {
                 const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 3c33eff3a..03ab766d9 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -5459,6 +5459,141 @@ private:
     bool   wav_header_sent = false;
 };

+// Moshi family (e.g. Hibiki): full duplex, the caller drives the backbone with the rows of an audio stream (mtmd_audio_stream)
+// per frame: the caller decodes the stream row + h_state_out of the previous step, samples the text token from it,
+// then step_gen() turns the text token and the hidden state into the codes of the frame (depformer) and decodes their audio
+// h_state_out is the backbone embedding of the generated codes, to be added to the next stream row (zero before the first step)
+class moshi_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
+public:
+    using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
+
+    // the acoustic codes lag the semantic one by this many frames, see "delays" of the checkpoint
+    static constexpr int acoustic_delay = 2;
+
+    int32_t max_frames() const override { return 0; }
+
+    void reset() override {
+        n_step = 0;
+        cb0_hist.clear();
+        audio_pcm.clear();
+        pcm_sent = 0;
+        h_state_buf.clear();
+        gen_state.reset();
+    }
+
+    int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
+        reset();
+        if (inp->prompt_len > 0 || inp->speaker_ref) {
+            LOG_ERR("mtmd_helper_gen_audio: the model answers an audio stream, it takes no prompt or speaker\n");
+            return 2;
+        }
+        if (!inp->stream || inp->out_type != MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+            LOG_ERR("mtmd_helper_gen_audio: the model only streams PCM\n");
+            return 2;
+        }
+        const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
+        temp  = inp->temp >= 0.0f ? inp->temp : def.temp;
+        top_k = def.top_k;
+        return init_gen_state(inp->seed) ? 0 : 1;
+    }
+
+    int32_t step_prompt(int32_t n_batch) override {
+        GGML_UNUSED(n_batch);
+        return 0;
+    }
+
+    bool next_is_text() const override {
+        return true;
+    }
+
+    int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
+        if (out_stop) {
+            *out_stop = false;
+        }
+        mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+        inp.type  = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
+        inp.embd  = const_cast<float *>(h_state_in);
+        inp.code0 = sampled;
+        inp.top_k = top_k;
+        inp.temp  = temp;
+        mtmd_gen_out out{};
+        if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_codes == 0) {
+            LOG_ERR("mtmd_helper_gen_audio: depformer failed\n");
+            return 1;
+        }
+        const std::vector<int32_t> codes(out.codes, out.codes + out.n_codes);
+        const int n_q = (int) codes.size();
+        h_state_buf.assign(out.embd, out.embd + n_embd);
+
+        // the next row is position n_step + 1, its acoustic codes are still the initial ones at the start
+        if (n_step + 1 <= acoustic_delay) {
+            std::vector<int32_t> fb = codes;
+            std::fill(fb.begin() + 1, fb.end(), code_initial);
+            inp.codes   = fb.data();
+            inp.n_codes = fb.size();
+            if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
+                LOG_ERR("mtmd_helper_gen_audio: code embedding failed\n");
+                return 1;
+            }
+            h_state_buf.assign(out.embd, out.embd + n_embd);
+        }
+
+        // frame f has the semantic code of step f and the acoustic codes of step f + acoustic_delay
+        cb0_hist.push_back(codes[0]);
+        if ((int) cb0_hist.size() > acoustic_delay) {
+            std::vector<int32_t> frame = codes;
+            frame[0] = cb0_hist.front();
+            cb0_hist.erase(cb0_hist.begin());
+
+            mtmd_gen_inp winp = mtmd_gen_inp_default(mctx);
+            winp.type    = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
+            winp.codes   = frame.data();
+            winp.n_codes = (size_t) n_q;
+            mtmd_gen_out wout{};
+            if (mtmd_gen_audio_process(mctx, gen_state.get(), &winp, &wout) != 0) {
+                LOG_ERR("mtmd_helper_gen_audio: mimi decode failed\n");
+                return 1;
+            }
+            audio_pcm.insert(audio_pcm.end(), wout.audio, wout.audio + wout.n_samples);
+        }
+
+        n_step++;
+        *h_state_out = h_state_buf.data();
+        return 0;
+    }
+
+    int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
+        *out_sample_rate = info.sample_rate;
+        // the audio delivered by the previous call is not needed again
+        audio_pcm.erase(audio_pcm.begin(), audio_pcm.begin() + pcm_sent);
+        if (out_n_samples) {
+            *out_n_samples = (int64_t) audio_pcm.size();
+        }
+        *out_data     = (const char *) audio_pcm.data();
+        *out_data_len = audio_pcm.size() * sizeof(float);
+        pcm_sent = audio_pcm.size();
+        return 0;
+    }
+
+    // every frame is decoded as soon as its codes are known
+    int32_t flush() override {
+        return 0;
+    }
+
+private:
+    // the extra entry after the mimi codebook, see _get_initial_token() of the reference
+    static constexpr int32_t code_initial = 2048;
+
+    float   temp  = 0.8f;
+    int32_t top_k = 250;
+
+    int64_t n_step = 0;
+    std::vector<int32_t> cb0_hist;    // semantic codes waiting for the acoustic codes of their frame
+    std::vector<float>   audio_pcm;
+    size_t               pcm_sent = 0;
+    std::vector<float>   h_state_buf;
+};
+
 static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
     switch (mtmd_gen_audio_get_info(mctx).type) {
         case MTMD_GEN_AUDIO_TYPE_QWEN3TTS:
@@ -5485,6 +5620,8 @@ static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lc
             return std::unique_ptr<mtmd_gen_audio_pipeline>(new music3_gen_audio_pipeline(lctx, mctx));
         case MTMD_GEN_AUDIO_TYPE_FISH_S2:
             return std::unique_ptr<mtmd_gen_audio_pipeline>(new fish_s2_gen_audio_pipeline(lctx, mctx));
+        case MTMD_GEN_AUDIO_TYPE_MOSHI:
+            return std::unique_ptr<mtmd_gen_audio_pipeline>(new moshi_gen_audio_pipeline(lctx, mctx));
         case MTMD_GEN_AUDIO_TYPE_VOXCPM2:
             return std::unique_ptr<mtmd_gen_audio_pipeline>(new voxcpm2_gen_audio_pipeline(lctx, mctx));
         default:
diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp
index bce73a74e..165936d80 100644
--- a/tools/mtmd/mtmd-helper.cpp
+++ b/tools/mtmd/mtmd-helper.cpp
@@ -1048,6 +1048,7 @@ mtmd_helper_cap mtmd_helper_get_capabilities(const llama_model * model, const mt
     cap.audio_gen = gen_audio_type != MTMD_GEN_AUDIO_TYPE_NONE;
     cap.audio_gen_spk_ref = gen_audio_type == MTMD_GEN_AUDIO_TYPE_QWEN3TTS
                          || gen_audio_type == MTMD_GEN_AUDIO_TYPE_POCKETTTS;
+    cap.audio_duplex = cap.audio_stream && gen_audio_type == MTMD_GEN_AUDIO_TYPE_MOSHI;

     // a tts model without a chat template only outputs audio, it cannot be used for chat
     // neither can a realtime transcription model (e.g. Voxtral Realtime): its text follows the audio by a token delay
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index 3fb75e5a3..c20cb0a20 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -260,6 +260,7 @@ struct mtmd_helper_cap {

     // realtime capabilities
     bool audio_stream; // realtime transcription, see mtmd_audio_stream_init()
+    bool audio_duplex; // full duplex (e.g. Hibiki): the model speaks while it listens, see the moshi pipeline of mtmd_helper_gen_audio

     // output capabilities
     bool text_gen; // false for an audio-only model (tts-only, no chat template)
diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h
index 582b11365..12bc9728d 100644
--- a/tools/mtmd/mtmd-impl.h
+++ b/tools/mtmd/mtmd-impl.h
@@ -614,6 +614,8 @@ enum projector_type {
     PROJECTOR_TYPE_QWEN3TTS_GEN,
     PROJECTOR_TYPE_POCKETTTS_SPKENC,
     PROJECTOR_TYPE_POCKETTTS_GEN,
+    PROJECTOR_TYPE_MOSHI,
+    PROJECTOR_TYPE_MOSHI_GEN,
     PROJECTOR_TYPE_SOPRANO,
     PROJECTOR_TYPE_KOKORO,
     PROJECTOR_TYPE_TINY_VOCA,
@@ -714,6 +716,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_QWEN3TTS_GEN,      "qwen3tts_gen"},
     { PROJECTOR_TYPE_POCKETTTS_SPKENC,  "pockettts_spkenc"},
     { PROJECTOR_TYPE_POCKETTTS_GEN,     "pockettts_gen"},
+    { PROJECTOR_TYPE_MOSHI,             "moshi"},
+    { PROJECTOR_TYPE_MOSHI_GEN,         "moshi_gen"},
     { PROJECTOR_TYPE_SOPRANO,           "soprano"},
     { PROJECTOR_TYPE_KOKORO,            "kokoro"},
     { PROJECTOR_TYPE_TINY_VOCA,         "tiny_voca"},
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index 2f4f40591..570750b9b 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -1151,6 +1151,20 @@ struct mtmd_model_loader {
                         hparams.flow_n_step       = 1;
                         hparams.gen_eos_threshold = -4.0f;
                     } break;
+                case PROJECTOR_TYPE_MOSHI:
+                case PROJECTOR_TYPE_MOSHI_GEN:
+                    {
+                        // mimi of the moshi family, see _seanet_kwargs and _transformer_kwargs of the reference
+                        hparams.audio_sample_rate = 24000;
+                        // seanet ratios are [8,6,5,4] in the config, the encoder reverses them
+                        hparams.seanet_ratios    = { 4, 5, 6, 8 };
+                        hparams.seanet_n_stage   = (int32_t) hparams.seanet_ratios.size();
+                        hparams.mimi_downsample  = 2;
+                        hparams.mimi_tfm_context = 250;
+                        hparams.rope_theta       = 10000.0f;
+                        // one step of the audio stream
+                        hparams.warmup_audio_size = 4 * mtmd_graph_moshi::n_samples_per_frame;
+                    } break;
                 case PROJECTOR_TYPE_VOXCPM2_SPKENC:
                     {
                         // AudioVAE encoder, one backbone row per patch of latents (checked against the strides at load)
@@ -1429,6 +1443,7 @@ struct mtmd_model_loader {
                                        model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
                                        model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
                                        model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC &&
+                                       model.proj_type != PROJECTOR_TYPE_MOSHI &&
                                        model.proj_type != PROJECTOR_TYPE_FISH_S2_SPKENC &&
                                        model.proj_type != PROJECTOR_TYPE_VOXCPM2_SPKENC;

@@ -1527,6 +1542,29 @@ struct mtmd_model_loader {
             }
         };

+        // mimi decoder: upsample, transformer, SEANet decoder
+        auto load_mimi_decoder = [&](int n_tfm_layer) {
+            model.gen_upsample_w = get_tensor(mtmd_format(TN_A_GEN_WAV_UPSAMPLE, "weight"));
+            load_seanet(model.seanet, true);
+            model.gen_tfm_layers.resize(n_tfm_layer);
+            for (int il = 0; il < n_tfm_layer; il++) {
+                auto & layer = model.gen_tfm_layers[il];
+                const char * p = "a.gen.wav.tfm";
+                layer.ln_1_w    = get_tensor(mtmd_format(TN_LN_1,        p, il, "weight"));
+                layer.ln_1_b    = get_tensor(mtmd_format(TN_LN_1,        p, il, "bias"));
+                layer.q_w       = get_tensor(mtmd_format(TN_ATTN_Q,      p, il, "weight"));
+                layer.k_w       = get_tensor(mtmd_format(TN_ATTN_K,      p, il, "weight"));
+                layer.v_w       = get_tensor(mtmd_format(TN_ATTN_V,      p, il, "weight"));
+                layer.o_w       = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"));
+                layer.ls_1_w    = get_tensor(mtmd_format(TN_LS_1,        p, il, "weight"));
+                layer.ln_2_w    = get_tensor(mtmd_format(TN_LN_2,        p, il, "weight"));
+                layer.ln_2_b    = get_tensor(mtmd_format(TN_LN_2,        p, il, "bias"));
+                layer.ff_up_w   = get_tensor(mtmd_format(TN_FFN_UP,      p, il, "weight"));
+                layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN,    p, il, "weight"));
+                layer.ls_2_w    = get_tensor(mtmd_format(TN_LS_2,        p, il, "weight"));
+            }
+        };
+
         auto get_vector = [&](const std::string & name) {
             std::vector<float> result;
             auto it = tensor_offset.find(name);
@@ -2759,25 +2797,7 @@ struct mtmd_model_loader {

                     // mimi decoder
                     model.gen_quant_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_OUT, "weight"));
-                    model.gen_upsample_w  = get_tensor(mtmd_format(TN_A_GEN_WAV_UPSAMPLE, "weight"));
-                    load_seanet(model.seanet, true);
-                    model.gen_tfm_layers.resize(hparams.n_layer);
-                    for (int il = 0; il < hparams.n_layer; il++) {
-                        auto & layer = model.gen_tfm_layers[il];
-                        const char * p = "a.gen.wav.tfm";
-                        layer.ln_1_w    = get_tensor(mtmd_format(TN_LN_1,        p, il, "weight"));
-                        layer.ln_1_b    = get_tensor(mtmd_format(TN_LN_1,        p, il, "bias"));
-                        layer.q_w       = get_tensor(mtmd_format(TN_ATTN_Q,      p, il, "weight"));
-                        layer.k_w       = get_tensor(mtmd_format(TN_ATTN_K,      p, il, "weight"));
-                        layer.v_w       = get_tensor(mtmd_format(TN_ATTN_V,      p, il, "weight"));
-                        layer.o_w       = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"));
-                        layer.ls_1_w    = get_tensor(mtmd_format(TN_LS_1,        p, il, "weight"));
-                        layer.ln_2_w    = get_tensor(mtmd_format(TN_LN_2,        p, il, "weight"));
-                        layer.ln_2_b    = get_tensor(mtmd_format(TN_LN_2,        p, il, "bias"));
-                        layer.ff_up_w   = get_tensor(mtmd_format(TN_FFN_UP,      p, il, "weight"));
-                        layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN,    p, il, "weight"));
-                        layer.ls_2_w    = get_tensor(mtmd_format(TN_LS_2,        p, il, "weight"));
-                    }
+                    load_mimi_decoder(hparams.n_layer);

                     for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) {
                         const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
@@ -2799,6 +2819,43 @@ struct mtmd_model_loader {
                         LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size());
                     }
                 } break;
+            case PROJECTOR_TYPE_MOSHI:
+                {
+                    // streaming mimi encoder, its RVQ, then the backbone embeddings of the input codes
+                    load_seanet(model.seanet, false);
+                    model.downsample_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
+                    auto & c2w = model.c2w;
+                    c2w.quant_first_in_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_IN, "weight"));
+                    c2w.quant_first_cb_w = get_tensor(mtmd_format(TN_A_CODEC_VQ_FIRST_CB, "weight"));
+                    c2w.quant_rest_in_w  = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_IN,  "weight"));
+                    c2w.quant_rest_cb_w  = get_tensor(mtmd_format(TN_A_CODEC_VQ_REST_CB,  "weight"));
+                    model.mm_a_code_embd = get_tensor(mtmd_format(TN_MM_A_CODE_EMBD, "weight"));
+                    if (model.mm_a_code_embd->ne[2] != c2w.quant_rest_cb_w->ne[2] + 1) {
+                        throw std::runtime_error(mtmd_format("%s: the input code embeddings do not match the codebooks\n", __func__));
+                    }
+                } break;
+            case PROJECTOR_TYPE_MOSHI_GEN:
+                {
+                    // depformer, the generic loop above loaded the a.gen.code.blk.* tensors (per-step weights on dim 2)
+                    model.gen_code_proj_in_w  = get_tensor(mtmd_format(TN_A_GEN_CODE_PROJ_IN,  "weight"));
+                    model.gen_code_tok_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_TOK_EMBD, "weight"));
+                    model.gen_code_embd_w     = get_tensor(mtmd_format(TN_A_GEN_CODE_EMBD,     "weight"));
+                    model.gen_code_head_w     = get_tensor(mtmd_format(TN_A_GEN_CODE_HEAD,     "weight"));
+                    model.gen_code_out_embd_w = get_tensor(mtmd_format(TN_A_GEN_CODE_OUT_EMBD, "weight"));
+                    const int64_t n_q = model.gen_code_head_w->ne[2];
+                    if (model.gen_code_embd_w->ne[2] != n_q - 1 || model.gen_code_out_embd_w->ne[2] != n_q ||
+                            model.gen_code_proj_in_w->ne[2] > n_q || (int64_t) n_q > mtmd_graph_moshi_gen::n_q_max) {
+                        throw std::runtime_error(mtmd_format("%s: invalid depformer tensors\n", __func__));
+                    }
+
+                    // RVQ decode, then the mimi decoder
+                    auto & c2w = model.c2w;
+                    c2w.quant_first_out_w = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight"));
+                    c2w.quant_first_cb_w  = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_FIRST_CB,  "weight"));
+                    c2w.quant_rest_out_w  = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_OUT,  "weight"));
+                    c2w.quant_rest_cb_w   = get_tensor(mtmd_format(TN_A_GEN_WAV_QUANT_REST_CB,   "weight"));
+                    load_mimi_decoder(mtmd_graph_moshi_gen::mimi_n_layer);
+                } break;
             case PROJECTOR_TYPE_LFM3A_GEN:
                 {
                     // must match the "tts" chat template, see conversion/lfm3.py
@@ -5023,6 +5080,16 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
                 // one latent per call for GEN_CODE, GEN_WAV sizes its input from the caller
                 n_patches = 1;
             } break;
+        case PROJECTOR_TYPE_MOSHI:
+            {
+                // one row per codebook and frame, then the rows of the initial codes, see mtmd_graph_moshi
+                const int n_q = (int) mm_a_code_embd->ne[2];
+                n_patches = n_q * (img->nx() / mtmd_graph_moshi::n_samples_per_frame + 1);
+            } break;
+        case PROJECTOR_TYPE_MOSHI_GEN:
+            {
+                n_patches = 1;
+            } break;
         case PROJECTOR_TYPE_GRANITE4_VISION:
             {
                 // Per-tile output token count: each projector block outputs
@@ -5214,6 +5281,10 @@ int mtmd_model::n_mmproj_embd() const {
             return spk_proj_w->ne[1];
         case PROJECTOR_TYPE_POCKETTTS_GEN:
             return gen_input_lin_w->ne[1];
+        case PROJECTOR_TYPE_MOSHI:
+            return mm_a_code_embd->ne[0];
+        case PROJECTOR_TYPE_MOSHI_GEN:
+            return gen_code_out_embd_w->ne[0];
         case PROJECTOR_TYPE_CHATTERBOX:
             // talker width, carried by the speech table
             return cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0];
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index aea0c1b92..078f154c7 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -392,6 +392,7 @@ static std::vector<c2w_state_slot> list_gen_state_slots(const mtmd_hparams & hpa
         case PROJECTOR_TYPE_FISH_S2_GEN:   return list_c2w_state_slots(hparams, model);
         case PROJECTOR_TYPE_DRAMA3_GEN:    return list_drama3_state_slots(hparams, model);
         case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model);
+        case PROJECTOR_TYPE_MOSHI_GEN:     return list_pockettts_state_slots(hparams, model);
         case PROJECTOR_TYPE_LFM3A_GEN:     return list_lfm3a_gen_state_slots(hparams, model);
         case PROJECTOR_TYPE_ZONOS3_GEN:    return list_zonos3_gen_state_slots(hparams, model);
         case PROJECTOR_TYPE_VOXCPM2_GEN:   return list_voxcpm2_state_slots(hparams, model);
@@ -405,6 +406,7 @@ static std::vector<c2w_state_slot> list_enc_state_slots(const mtmd_hparams & hpa
         case PROJECTOR_TYPE_LFM3A:   return list_lfm3a_state_slots(hparams, model);
         case PROJECTOR_TYPE_GEMMA4A: return list_gemma4a_state_slots(hparams, model);
         case PROJECTOR_TYPE_VIBEVOICE_ASR: return list_vibevoice_asr_state_slots(hparams, model);
+        case PROJECTOR_TYPE_MOSHI:   return list_moshi_state_slots(hparams, model);
         default:                     return {};
     }
 }
@@ -454,7 +456,8 @@ mtmd_subctx_gen_state * mtmd_subcontext::gen_state_init(uint32_t seed) {

 mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) {
     const auto & hparams = model.hparams;
-    if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR) {
+    if (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR ||
+            model.proj_type == PROJECTOR_TYPE_MOSHI) {
         // see list_enc_state_slots()
         const auto slots = list_enc_state_slots(hparams, model);
         auto st = std::make_unique<mtmd_subctx_enc_state>();
@@ -2329,6 +2332,61 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
                 }
                 set_input_f32("pos_emb", pos_emb);
             } break;
+        case PROJECTOR_TYPE_MOSHI:
+            {
+                // positions and masks are derived in-graph from the "pos" slot
+                set_enc_state_in();
+            } break;
+        case PROJECTOR_TYPE_MOSHI_GEN:
+            {
+                const int64_t n_q = model.gen_code_head_w->ne[2];
+                if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
+                    GGML_ASSERT(params->codes != nullptr);
+                    // frame-major to codebook-major
+                    const auto &  in       = *params->codes;
+                    const int64_t n_frames = (int64_t) in.size() / n_q;
+                    const int64_t n_codes  = model.c2w.quant_first_cb_w->ne[1];
+                    std::vector<int32_t> codes(in.size());
+                    for (int64_t f = 0; f < n_frames; f++) {
+                        for (int64_t k = 0; k < n_q; k++) {
+                            const int32_t c = in[(size_t) (f * n_q + k)];
+                            if (c < 0 || c >= n_codes) {
+                                LOG_ERR("%s: code out of range (%d, codebook size %lld)\n", __func__, c, (long long) n_codes);
+                                return false;
+                            }
+                            codes[(size_t) (k * n_frames + f)] = c;
+                        }
+                    }
+                    set_input_i32("inp_codes", codes);
+                    set_gen_state_in();
+                } else if (params->codes != nullptr) {
+                    std::vector<int32_t> codes = *params->codes;
+                    const int64_t n_rows = model.gen_code_out_embd_w->ne[1];
+                    for (int32_t c : codes) {
+                        if (c < 0 || c >= n_rows) {
+                            LOG_ERR("%s: code out of range (%d)\n", __func__, c);
+                            return false;
+                        }
+                    }
+                    set_input_i32("inp_codes", codes);
+                } else {
+                    std::vector<int32_t> code0 = { params->code0 };
+                    if (code0[0] < 0 || code0[0] >= model.gen_code_tok_embd_w->ne[1]) {
+                        LOG_ERR("%s: text token out of range (%d)\n", __func__, code0[0]);
+                        return false;
+                    }
+                    set_input_i32("inp_code0", code0);
+                    // greedy has no random input
+                    if (ggml_graph_get_tensor(gf, "inp_rand") != nullptr) {
+                        std::uniform_real_distribution<float> dist(0.0f, 1.0f);
+                        std::vector<float> r(n_q);
+                        for (auto & v : r) {
+                            v = dist(params->gen_state->rng);
+                        }
+                        set_input_f32("inp_rand", r);
+                    }
+                }
+            } break;
         case PROJECTOR_TYPE_LFM3A:
             {
                 GGML_ASSERT(imgs.entries.size() == 1);
@@ -2902,7 +2960,8 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
             }
         }
     }
-    if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR)) {
+    if (params->enc_state != nullptr && (model.proj_type == PROJECTOR_TYPE_LFM3A || model.proj_type == PROJECTOR_TYPE_GEMMA4A || model.proj_type == PROJECTOR_TYPE_VIBEVOICE_ASR ||
+                                         model.proj_type == PROJECTOR_TYPE_MOSHI)) {
         const auto & slots = params->enc_state->slots;
         size_t i = 0;
         for (const auto & slot : list_enc_state_slots(hparams, model)) {
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 666f48eeb..fdac762ac 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -1470,6 +1470,7 @@ struct mtmd_context {
                     audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(models.a.get());
                 } break;
             case PROJECTOR_TYPE_VOXTRAL_RT:
+            case PROJECTOR_TYPE_MOSHI:
                 {
                     // audio and text share positions, only usable via mtmd_audio_stream
                 } break;
@@ -2827,6 +2828,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
             info.type = MTMD_GEN_AUDIO_TYPE_POCKETTTS;
             info.sample_rate = 24000;
             break;
+        case PROJECTOR_TYPE_MOSHI_GEN:
+            info.type = MTMD_GEN_AUDIO_TYPE_MOSHI;
+            info.sample_rate = 24000;
+            break;
         case PROJECTOR_TYPE_SOPRANO:
             info.type = MTMD_GEN_AUDIO_TYPE_SOPRANO;
             info.sample_rate = 32000;
@@ -3344,6 +3349,12 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
         case PROJECTOR_TYPE_TINY_VOCA:
             inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
             break;
+        case PROJECTOR_TYPE_MOSHI_GEN:
+            // lm_gen_config of the checkpoint (audio codes), the text token is sampled by the caller
+            inp.top_k = 250;
+            inp.top_p = 1.0f;
+            inp.temp  = 0.8f;
+            break;
         case PROJECTOR_TYPE_VOXCPM2_GEN:
             // UnifiedCFM.forward(): unit gaussian noise, see VoxCPM2Model._inference() for the steps and the guidance
             inp.temp      = 1.0f;
@@ -5485,6 +5496,36 @@ struct mtmd_audio_stream_lfm3a : mtmd_audio_stream {
     int32_t step_live(llama_token last_token);
 };

+// Moshi family (e.g. Hibiki): full duplex, one row per 80 ms frame of input audio
+// row = embeddings of the input codes (the acoustic ones lag by acoustic_delay frames) + the text token sampled from the previous row
+// the codes the model generates are added by the caller, see the moshi pipeline of mtmd_helper_gen_audio
+struct mtmd_audio_stream_moshi : mtmd_audio_stream {
+    static constexpr int acoustic_delay = 2;
+    static constexpr int n_frames_max   = 4; // frames per encoder call
+
+    mtmd_subctx_enc_state * enc_state = nullptr;
+
+    int n_q = 0;
+    llama_token tok_text_start = LLAMA_TOKEN_NULL;
+
+    std::vector<float> codes_embd; // n_q rows per encoded frame, from frame codes_off
+    int64_t codes_off = 0;
+    int64_t n_frames_enc = 0;
+    std::vector<float> init_embd;  // n_q rows of the initial codes
+    std::vector<float> tok_embd;
+
+    ~mtmd_audio_stream_moshi() override {
+        delete enc_state;
+    }
+
+    bool    init(const mtmd_audio_stream_params & params) override;
+    int32_t flush(bool end) override;
+    int32_t step(llama_token last_token) override;
+
+    bool encode();
+    const float * code_row(int64_t frame, int k) const;
+};
+
 // Gemma 4: the encoder is causal
 struct mtmd_audio_stream_gemma4a : mtmd_audio_stream {
     mtmd_subctx_enc_state * enc_state = nullptr;
@@ -5564,7 +5605,7 @@ bool mtmd_support_audio_stream(const mtmd_context * ctx) {
     }
     const projector_type proj = ctx->ctx_a->model.proj_type;
     return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A || proj == PROJECTOR_TYPE_GEMMA4A ||
-           proj == PROJECTOR_TYPE_HIGGS_AUDIO || proj == PROJECTOR_TYPE_VIBEVOICE_ASR;
+           proj == PROJECTOR_TYPE_HIGGS_AUDIO || proj == PROJECTOR_TYPE_VIBEVOICE_ASR || proj == PROJECTOR_TYPE_MOSHI;
 }

 mtmd_audio_stream_params mtmd_audio_stream_params_default(void) {
@@ -5589,6 +5630,7 @@ mtmd_audio_stream * mtmd_audio_stream_init(mtmd_context * ctx, mtmd_audio_stream
         case PROJECTOR_TYPE_GEMMA4A:    s = std::make_unique<mtmd_audio_stream_gemma4a>();    break;
         case PROJECTOR_TYPE_HIGGS_AUDIO:   s = std::make_unique<mtmd_audio_stream_higgs_audio>();   break;
         case PROJECTOR_TYPE_VIBEVOICE_ASR: s = std::make_unique<mtmd_audio_stream_vibevoice_asr>(); break;
+        case PROJECTOR_TYPE_MOSHI:         s = std::make_unique<mtmd_audio_stream_moshi>();         break;
         default:                        GGML_ABORT("unsupported projector");
     }
     s->ctx      = ctx;
@@ -6538,3 +6580,120 @@ int64_t mtmd_audio_stream_get_n_rows(const mtmd_audio_stream * s) {
 int64_t mtmd_audio_stream_get_n_rows_flushed(const mtmd_audio_stream * s) {
     return s->n_rows_flushed;
 }
+
+//
+// Moshi
+//
+
+bool mtmd_audio_stream_moshi::init(const mtmd_audio_stream_params & params) {
+    if (params.n_prompt > 0) {
+        LOG_ERR("%s: moshi takes no prompt\n", __func__);
+        return false;
+    }
+    n_q = (int) sctx->model.mm_a_code_embd->ne[2];
+    samples_per_tok = mtmd_graph_moshi::n_samples_per_frame;
+
+    const std::string name = "<|text_start|>";
+    std::vector<llama_token> toks(4);
+    if (llama_tokenize(ctx->vocab, name.c_str(), (int32_t) name.size(), toks.data(), (int32_t) toks.size(), false, true) != 1) {
+        LOG_ERR("%s: cannot find the %s token\n", __func__, name.c_str());
+        return false;
+    }
+    tok_text_start = toks[0];
+
+    enc_state = sctx->enc_state_init(0);
+    return enc_state != nullptr;
+}
+
+int32_t mtmd_audio_stream_moshi::flush(bool end) {
+    // pad to a whole frame
+    const int64_t n_align = (samples_per_tok - n_samples % samples_per_tok) % samples_per_tok;
+    add_silence(n_align);
+    n_rows_flushed = n_samples / samples_per_tok;
+    ended = end;
+    return 0;
+}
+
+// encodes the whole frames of buf, n_frames_max per call
+bool mtmd_audio_stream_moshi::encode() {
+    while (true) {
+        const int64_t n = std::min<int64_t>((int64_t) buf.size() / samples_per_tok, n_frames_max);
+        if (n <= 0) {
+            return true;
+        }
+        const size_t n_in = (size_t) (n * samples_per_tok);
+
+        mtmd_image_f32 img;
+        img.set_size({(int) n_in, 1}, false, /* is_audio */ true);
+        img.cpy_buf(std::vector<float>(buf.begin(), buf.begin() + n_in));
+        mtmd_image_f32_batch batch;
+        batch.is_audio = true;
+        batch.entries.push_back(std::move(img));
+
+        std::vector<float> res((size_t) n_embd * n_q * (n + 1));
+        mtmd_subctx_encode_params params;
+        params.n_threads = ctx->n_threads;
+        params.imgs      = &batch;
+        params.out_embd  = &res;
+        params.enc_state = enc_state;
+        if (!sctx->encode(&params)) {
+            LOG_ERR("%s: failed to encode audio\n", __func__);
+            return false;
+        }
+        const size_t n_codes = (size_t) n_embd * n_q * n;
+        codes_embd.insert(codes_embd.end(), res.begin(), res.begin() + n_codes);
+        init_embd.assign(res.begin() + n_codes, res.end());
+        n_frames_enc += n;
+
+        buf.erase(buf.begin(), buf.begin() + n_in);
+        buf_pos += (int64_t) n_in;
+    }
+}
+
+const float * mtmd_audio_stream_moshi::code_row(int64_t frame, int k) const {
+    return codes_embd.data() + ((size_t) (frame - codes_off) * n_q + k) * n_embd;
+}
+
+int32_t mtmd_audio_stream_moshi::step(llama_token last_token) {
+    n_rows_done = n_rows;
+    out.clear();
+
+    if (!encode()) {
+        return -2;
+    }
+    // row p goes with frame p
+    if (n_rows >= n_frames_enc) {
+        return ended ? -1 : 0;
+    }
+    const int64_t p = n_rows;
+
+    // the text of the previous row, see _get_initial_token() of the reference
+    const llama_token tok = p == 0 ? tok_text_start : last_token;
+    if (tok == LLAMA_TOKEN_NULL) {
+        LOG_ERR("%s: the token sampled from the previous row is required\n", __func__);
+        return -2;
+    }
+    tok_embd.resize(n_embd);
+    if (llama_model_get_tok_embd_rows(ctx->text_model, &tok, 1, tok_embd.data()) == 0) {
+        return -2;
+    }
+
+    // the semantic code has no delay; a stream starts with initial codes, the first frame is never seen, like the reference
+    out = tok_embd;
+    for (int k = 0; k < n_q; k++) {
+        const int64_t delay = k == 0 ? 0 : acoustic_delay;
+        const float * row = p <= delay ? init_embd.data() + (size_t) k * n_embd : code_row(p - delay, k);
+        for (int i = 0; i < n_embd; i++) {
+            out[i] += row[i];
+        }
+    }
+    n_rows++;
+
+    // drop the frames no row will need again
+    const int64_t keep_from = std::max<int64_t>(0, n_rows - acoustic_delay);
+    if (keep_from - codes_off >= 64) {
+        codes_embd.erase(codes_embd.begin(), codes_embd.begin() + (size_t) (keep_from - codes_off) * n_q * n_embd);
+        codes_off = keep_from;
+    }
+    return 1;
+}
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index cf2d14ce2..09cae0ad8 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -478,6 +478,7 @@ enum mtmd_gen_audio_type {
     MTMD_GEN_AUDIO_TYPE_YUE2,
     MTMD_GEN_AUDIO_TYPE_MINIMAX_MUSIC3,
     MTMD_GEN_AUDIO_TYPE_FISH_S2,
+    MTMD_GEN_AUDIO_TYPE_MOSHI,
     MTMD_GEN_AUDIO_TYPE_KOKORO,
     MTMD_GEN_AUDIO_TYPE_TINY_VOCA,
     MTMD_GEN_AUDIO_TYPE_VOXCPM2,
@@ -534,6 +535,7 @@ struct mtmd_gen_inp {
     // for MTMD_GEN_PROCESS_TYPE_GEN_WAV
     // pass either codes (discrete) or feats (continuous), depending on the pipeline
     // lfm3a GEN_CODE also takes codes: the codes revealed so far, -1 for a masked codebook
+    // moshi GEN_CODE: code0 is the sampled text token; with codes (all of the frame), only their backbone embedding is built
     // zonos3 GEN_WAV: -1 for a residual code not known yet, the frame is decoded without it
     // kokoro GEN_WAV takes both: codes are the phoneme tokens, feats is the voice row (256 values) then the backbone state of each token
     int32_t * codes;
diff --git a/tools/server/README.md b/tools/server/README.md
index 1f6553389..b0168b6d1 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1667,6 +1667,12 @@ LFM3-Omni is not trained to reason, run it with `--reasoning off` so that its tu

 Tools go into the system prompt with the chat template, and the tool calls are parsed as in `/v1/chat/completions`. The client runs the calls of a response, sends their results as `function_call_output` items, then asks for the next response.

+#### Full duplex
+
+Full duplex models, like Hibiki (French to English speech translation), speak while they listen on the same endpoint. There are no turns: the first `input_audio_buffer.append` opens a response, then every 80 ms of input audio gives 80 ms of output audio (`response.output_audio.delta`) and the text of the translation (`response.output_text.delta`). `response.cancel` ends the response with `response.done`, the next audio opens another one. Audio input and output are PCM16 mono, as for a realtime conversation.
+
+The model attends to the last 120 s of the stream, so the context of each slot only needs to hold that window (1500 frames for Hibiki, `-c` divided by `-np`). The model is trained on streams no longer than its window, so once the window is full the first frames of the stream stay in context and the oldest frames after them are dropped, 20 s at a time. This keeps Hibiki-1B going on long streams; Hibiki-2B still degrades after about 2 minutes, as the reference implementation does, cancel the response between long segments of speech. `input_audio_buffer.commit`, `conversation.item.create` and `response.create` are rejected. The text and the audio are sampled with the settings of the model, the text settings can be overridden with the server sampling options.
+
 ### POST `/v1/audio/speech`: OpenAI-compatible Speech API

 Generates speech from text, for models that can generate audio. The request is converted to a [`/tts`](#post-tts-generate-speech-audio-from-text) request, so the `/tts` options such as `lang`, `seed` or `speaker_ref_b64` can also be used.
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index bfade25d7..e37661a9a 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -6132,7 +6132,7 @@ void server_routes::init_routes() {
         // models that answer after the audio hold a conversation, see server-realtime.h
         // live transcription: Voxtral Realtime (no chat), or an "asr" model in a session opened with ?intent=transcription
         const bool live_asr = !ctx_server.asr_prompt.empty() && req.get_param("intent") == "transcription";
-        if (ctx_server.mtmd_cap.audio_stream && ctx_server.mtmd_cap.chat && !live_asr) {
+        if (ctx_server.mtmd_cap.audio_duplex || (ctx_server.mtmd_cap.audio_stream && ctx_server.mtmd_cap.chat && !live_asr)) {
             server_realtime_conversation({queue_tasks, queue_results, *meta, params, ctx_server.mctx, ctx_server.mtmd_cap, ctx_server.vocab}, conn);
             return;
         }
diff --git a/tools/server/server-realtime.cpp b/tools/server/server-realtime.cpp
index 6f08120dd..b379fa203 100644
--- a/tools/server/server-realtime.cpp
+++ b/tools/server/server-realtime.cpp
@@ -13,7 +13,9 @@

 #include <algorithm>
 #include <atomic>
+#include <chrono>
 #include <climits>
+#include <condition_variable>
 #include <cmath>
 #include <cstring>
 #include <map>
@@ -180,8 +182,14 @@ struct realtime_conversation {

     std::atomic<bool> closed{false};

+    // full duplex: no turns, the first audio opens a response that runs until response.cancel
+    const bool              duplex;
+    bool                    duplex_open = false; // guarded by mutex
+    std::condition_variable duplex_done;         // the cancelled response is done
+
     realtime_conversation(const server_realtime_deps & deps, server_http_context::ws_conn & conn)
-        : deps(deps), conn(conn), out_language(deps.params.tts_lang), out_voice(deps.params.tts_voice) {}
+        : deps(deps), conn(conn), out_language(deps.params.tts_lang), out_voice(deps.params.tts_voice),
+          duplex(deps.cap.audio_duplex) {}

     //
     // events
@@ -395,6 +403,23 @@ struct realtime_conversation {
         if (samples.empty()) {
             return;
         }
+        if (duplex) {
+            if (!resample(samples) || samples.empty()) {
+                return;
+            }
+            bool open = false;
+            {
+                std::lock_guard<std::mutex> lock(mutex);
+                open = !duplex_open;
+                duplex_open = true;
+            }
+            if (open) {
+                started = true;
+                open_response(MTMD_HELPER_GEN_AUDIO_OUT_MIXED);
+            }
+            control(SERVER_CONTROL_ACTION_AUDIO_APPEND, {}, std::move(samples));
+            return;
+        }
         if (!resample(samples) || samples.empty() || !ensure_turn()) {
             return;
         }
@@ -449,6 +474,10 @@ struct realtime_conversation {
     }

     void on_commit() {
+        if (duplex) {
+            send_error("input_audio_buffer.commit: the model answers while it listens, there is no turn to commit");
+            return;
+        }
         const int64_t n_audio = n_turn_samples;
         if (!commit() || type != "transcription") {
             return;
@@ -480,6 +509,10 @@ struct realtime_conversation {
     }

     void on_item_create(const json & ev) {
+        if (duplex) {
+            send_error("conversation.item.create: the model only takes audio");
+            return;
+        }
         if (type == "transcription") {
             send_error("conversation.item.create: not supported in a transcription session");
             return;
@@ -537,6 +570,10 @@ struct realtime_conversation {
     }

     void on_response_create(const json & ev) {
+        if (duplex) {
+            send_error("response.create: the model answers while it listens, its response opens with the first audio");
+            return;
+        }
         if (type == "transcription") {
             send_error("response.create: not supported in a transcription session");
             return;
@@ -606,24 +643,7 @@ struct realtime_conversation {
         }
         response.sampling = response_sampling(chat ? &*chat : nullptr);

-        {
-            std::lock_guard<std::mutex> lock(mutex);
-            response_id      = "resp_" + random_string();
-            response_item_id = "item_" + random_string();
-            response_text.clear();
-            parts       = json::array();
-            idx_text    = -1;
-            idx_audio   = -1;
-            output      = json::array();
-            idx_message = -1;
-            call_items.clear();
-            parse.reset();
-            if (chat) {
-                parse.emplace(std::move(*chat));
-            }
-            send_event({{"type", "response.created"}, {"response", {{"id", response_id}, {"object", "realtime.response"},
-                        {"status", "in_progress"}, {"output_modalities", output_modalities_json(mode)}, {"output", json::array()}}}});
-        }
+        open_response(mode, chat ? &*chat : nullptr);
         if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, prompt, {}, response)) {
             return;
         }
@@ -634,6 +654,26 @@ struct realtime_conversation {
         n_turns++;
     }

+    // the results that follow belong to a new response
+    void open_response(mtmd_helper_gen_audio_out_mode mode, common_chat_session * chat = nullptr) {
+        std::lock_guard<std::mutex> lock(mutex);
+        response_id      = "resp_" + random_string();
+        response_item_id = "item_" + random_string();
+        response_text.clear();
+        parts       = json::array();
+        idx_text    = -1;
+        idx_audio   = -1;
+        output      = json::array();
+        idx_message = -1;
+        call_items.clear();
+        parse.reset();
+        if (chat) {
+            parse.emplace(std::move(*chat));
+        }
+        send_event({{"type", "response.created"}, {"response", {{"id", response_id}, {"object", "realtime.response"},
+                    {"status", "in_progress"}, {"output_modalities", output_modalities_json(mode)}, {"output", json::array()}}}});
+    }
+
     void on_session_update(const json & ev) {
         const json session = ev.value("session", json::object());
         if (!session.is_object()) {
@@ -646,6 +686,10 @@ struct realtime_conversation {
                 send_error("session.update: unknown session type " + t);
                 return;
             }
+            if (duplex) {
+                send_error("session.update: the model only holds \"realtime\" sessions");
+                return;
+            }
             if (started) {
                 send_error("session.update: the session type cannot change after the first audio or message");
                 return;
@@ -726,7 +770,7 @@ struct realtime_conversation {
             out_voice    = voice;
             out_language = lang;
         }
-        if (!started && !configure()) {
+        if (!duplex && !started && !configure()) {
             send_error("the chat template does not fit the realtime turn format", "server_error");
             return;
         }
@@ -774,6 +818,11 @@ struct realtime_conversation {
             on_response_create(ev);
         } else if (t == "response.cancel") {
             control(SERVER_CONTROL_ACTION_RESPONSE_CANCEL);
+            if (duplex) {
+                // the next audio opens another response, once the results of this one are out
+                std::unique_lock<std::mutex> lock(mutex);
+                duplex_done.wait_for(lock, std::chrono::seconds(10), [this]() { return !duplex_open || closed.load(); });
+            }
         } else {
             send_error("unknown event type: " + t);
         }
@@ -918,6 +967,8 @@ struct realtime_conversation {
         send_event({{"type", "response.done"}, {"response", {{"id", response_id}, {"object", "realtime.response"}, {"status", status},
                     {"output", output}, {"usage", {{"output_tokens", res.n_decoded}}}}}});
         parse.reset();
+        duplex_open = false;
+        duplex_done.notify_all();
     }

     //
@@ -929,7 +980,7 @@ struct realtime_conversation {
         if (deps.cap.audio_gen) {
             output_rate = mtmd_gen_audio_get_info(deps.mctx).sample_rate;
         }
-        if (!configure()) {
+        if (!duplex && !configure()) {
             send_error("the chat template does not fit the realtime turn format", "server_error");
             conn.close();
             return;
@@ -1015,6 +1066,7 @@ server_realtime_slot::server_realtime_slot(llama_context * ctx, mtmd_context * m
     if (mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE) {
         gen.init(ctx, mctx);
     }
+    duplex = mtmd_helper_get_capabilities(model, mctx).audio_duplex;
 }

 void server_realtime_slot::push(server_realtime_op && op) {
@@ -1022,10 +1074,19 @@ void server_realtime_slot::push(server_realtime_op && op) {
 }

 void server_realtime_slot::cancel() {
+    if (duplex) {
+        // in order with the audio: the response of a full duplex model may still be opening
+        ops.push_back({SERVER_CONTROL_ACTION_RESPONSE_CANCEL, {}, {}, {}});
+        return;
+    }
     cancelled = responding;
 }

 bool server_realtime_slot::is_idle() const {
+    if (duplex) {
+        // a full duplex response waits for the next audio
+        return ops.empty() && !(responding && duplex_pending);
+    }
     return ops.empty() && !responding;
 }

@@ -1051,7 +1112,8 @@ void server_realtime_slot::send_result(common_chat_input content, std::vector<fl
 bool server_realtime_slot::decode(const llama_token * tokens, const float * rows, int32_t n, bool logits_last) {
     const int32_t n_embd  = llama_model_n_embd_inp(model);
     const int32_t n_batch = llama_n_batch(ctx);
-    if (n_past + n >= n_ctx) {
+    // a full duplex model attends to a sliding window, its stream has no end
+    if (!duplex && n_past + n >= n_ctx) {
         return fail("the conversation does not fit in the context", ERROR_TYPE_EXCEED_CONTEXT_SIZE);
     }
     common_batch batch(ctx);
@@ -1220,16 +1282,149 @@ bool server_realtime_slot::response_step() {
             // the pipeline decoded the response, the sequence ends where it stopped
             n_past = llama_memory_seq_pos_max(llama_get_memory(ctx), seq_id) + 1;
         }
-        send_result({}, {}, true);
-        responding = false;
-        cancelled  = false;
+        response_end();
+    }
+    return true;
+}
+
+void server_realtime_slot::response_end() {
+    send_result({}, {}, true);
+    responding = false;
+    cancelled  = false;
+}
+
+//
+// full duplex (e.g. Hibiki): one row per frame of input audio, the model speaks while it listens
+// row = the stream row (input audio + previous text token) + the embedding of the audio generated at the previous frame
+//
+
+bool server_realtime_slot::duplex_start() {
+    // the cells past the window are reused, the window itself must fit
+    const int32_t n_swa = llama_model_n_swa(model);
+    if (n_ctx < n_swa) {
+        return fail("the context of a slot (" + std::to_string(n_ctx) + ") must hold the attention window of the model (" +
+                    std::to_string(n_swa) + "), see -c and -np");
+    }
+    stream.reset(mtmd_audio_stream_init(mctx, mtmd_audio_stream_params_default()));
+    if (!stream) {
+        return fail("failed to initialize the audio stream");
+    }
+    mtmd_helper::gen_audio::inp inp;
+    inp.data.seq_id   = seq_id;
+    inp.data.seed     = sampling.seed;
+    inp.data.stream   = true;
+    inp.data.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM;
+    if (gen.set_input(inp.get()) != 0 || gen.step_prompt(llama_n_batch(ctx)) != 0) {
+        return fail("failed to start the response");
+    }
+    llama_memory_seq_rm(llama_get_memory(ctx), seq_id, -1, -1);
+    n_past    = 0;
+    n_decoded = 0;
+    sampled   = LLAMA_TOKEN_NULL;
+    h_state.assign(llama_model_n_embd_inp(model), 0.0f);
+    smpl.reset(common_sampler_init(model, sampling));
+    responding = true;
+    cancelled  = false;
+    return true;
+}
+
+bool server_realtime_slot::duplex_frame() {
+    const int32_t n = mtmd_audio_stream_step(stream.get(), sampled);
+    if (n == -2) {
+        return fail("audio stream error");
+    }
+    if (n <= 0) {
+        duplex_pending = false;
+        return true;
+    }
+    GGML_ASSERT(n == 1);
+
+    // the model was trained on streams no longer than its window, so it always saw their start (it acts as an attention sink)
+    // once the window is full: the first rows stay, the oldest rows after them go and the rest shifts back, like StreamingLLM
+    llama_memory_t mem   = llama_get_memory(ctx);
+    const int32_t  n_swa = llama_model_n_swa(model);
+    if (n_swa > 0 && n_past >= n_swa && llama_memory_can_shift(mem)) {
+        const int32_t n_discard = n_swa / duplex_discard_div;
+        llama_memory_seq_rm (mem, seq_id, duplex_n_keep, duplex_n_keep + n_discard);
+        llama_memory_seq_add(mem, seq_id, duplex_n_keep + n_discard, -1, -n_discard);
+        n_past -= n_discard;
+    }
+
+    const float * row = mtmd_audio_stream_get_output_embd(stream.get());
+    std::vector<float> inp(row, row + h_state.size());
+    for (size_t i = 0; i < inp.size(); i++) {
+        inp[i] += h_state[i];
+    }
+    if (!decode(nullptr, inp.data(), 1, true)) {
+        return false;
+    }
+    sampled = common_sampler_sample(smpl.get(), ctx, -1);
+    common_sampler_accept(smpl.get(), sampled, true);
+
+    // copy the hidden state before any other slot decodes
+    const float * h = llama_get_embeddings_ith(ctx, -1);
+    std::vector<float> h_cur(h, h + llama_model_n_embd(model));
+    const float * fb = nullptr;
+    if (gen.step_gen(sampled, h_cur.data(), &fb) != 0 || fb == nullptr) {
+        return fail("response generation failed");
+    }
+    h_state.assign(fb, fb + h_state.size());
+    n_decoded++;
+
+    common_chat_input content;
+    content.append(piece(sampled), sampled);
+    std::vector<float> audio;
+    int32_t sample_rate = 0;
+    const char * data   = nullptr;
+    size_t data_len     = 0;
+    if (gen.get_output(&sample_rate, &data, &data_len) == 0 && data_len > 0) {
+        audio.assign((const float *) data, (const float *) data + data_len / sizeof(float));
+    }
+    if (!content.empty() || !audio.empty()) {
+        send_result(std::move(content), std::move(audio));
     }
     return true;
 }

+bool server_realtime_slot::duplex_update() {
+    while (!ops.empty()) {
+        server_realtime_op op = std::move(ops.front());
+        ops.pop_front();
+        switch (op.action) {
+            case SERVER_CONTROL_ACTION_AUDIO_APPEND:
+                {
+                    // the first audio opens the response
+                    if (!responding && !duplex_start()) {
+                        return false;
+                    }
+                    mtmd_audio_stream_push(stream.get(), op.audio.data(), op.audio.size());
+                    duplex_pending = true;
+                } break;
+            case SERVER_CONTROL_ACTION_RESPONSE_CANCEL:
+                {
+                    // the audio after it opens another response
+                    if (responding) {
+                        stream.reset();
+                        gen.reset();
+                        duplex_pending = false;
+                        cancelled      = true;
+                        response_end();
+                    }
+                } break;
+            default:
+                break;
+        }
+    }
+    return !responding || !duplex_pending || duplex_frame();
+}
+
 bool server_realtime_slot::update() {
     llama_set_embeddings(ctx, gen.valid());

+    if (duplex) {
+        return duplex_update();
+    }
+
     if (responding) {
         return response_step();
     }
diff --git a/tools/server/server-realtime.h b/tools/server/server-realtime.h
index 93c864df2..cb0efa939 100644
--- a/tools/server/server-realtime.h
+++ b/tools/server/server-realtime.h
@@ -32,6 +32,7 @@ struct server_realtime_deps {

 // a realtime conversation over websocket, for models that answer after the audio (e.g. LFM3-Audio):
 // the audio is prefilled as it arrives, responses follow output_modalities or the model's own switching
+// full duplex models (e.g. Hibiki) answer while they listen: the first audio opens a response that runs until it is cancelled
 void server_realtime_conversation(const server_realtime_deps & deps, server_http_context::ws_conn & conn);

 // one control of a realtime conversation
@@ -80,12 +81,19 @@ private:
     int32_t     n_decoded  = 0;
     llama_token sampled    = LLAMA_TOKEN_NULL; // next text token of the response, sampled right after the slot's own decode

-    mtmd::audio_stream_ptr stream;   // the audio of the open turn
+    mtmd::audio_stream_ptr stream;   // the audio of the open turn, or of the whole response of a full duplex model
     mtmd_helper::gen_audio gen;      // set if the model generates audio
     std::vector<float>     h_state;  // hidden state that the gen-audio pipeline steps from
     common_sampler_ptr     smpl;
     std::set<llama_token>  preserved; // special tokens of the response that go out as text (tool calls)

+    // full duplex: one frame per update(), h_state holds the embedding of the generated audio for the next row
+    bool duplex         = false;
+    bool duplex_pending = false; // the stream may have a row ready
+    // past the attention window: the first rows of the stream stay, 1/duplex_discard_div of the window goes at once
+    static constexpr int32_t duplex_n_keep      = 3;
+    static constexpr int32_t duplex_discard_div = 6;
+
     bool fail(const std::string & msg, enum error_type type = ERROR_TYPE_SERVER);
     void send_result(common_chat_input content, std::vector<float> audio, bool response_done = false);
     bool decode(const llama_token * tokens, const float * rows, int32_t n, bool logits_last);
@@ -94,4 +102,9 @@ private:
     std::string piece(llama_token tok) const;
     bool response_start(const server_realtime_op & op);
     bool response_step();
+    void response_end();
+
+    bool duplex_update();
+    bool duplex_start();
+    bool duplex_frame();
 };