Commit d3064fddc for llama.cpp

commit d3064fddce83046ef66eded524cf00947d5166f3
Author: Aman Gupta <amangupta052@gmail.com>
Date:   Tue Nov 17 22:25:01 2026 +0800

    convert : fix FP8 qwen4exp checkpoints (n-gram table scale, block-scaled FP8) (#32476)

    * convert : fix FP8 n-gram table export for qwen4exp

    * convert : dequantize block-scaled FP8 weight_scale_inv in ModelOpt checkpoints

diff --git a/conversion/base.py b/conversion/base.py
index a95ecc008..7cfbff321 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -653,6 +653,28 @@ class ModelBase:
                     raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported")
             elif quant_method in ("modelopt", "modelopt_mixed"):
                 dequant_fp8()
+                # FP8_PB_WO: one scale per block, the block size follows from the two shapes
+                for name in self.model_tensors.keys():
+                    if not name.endswith(".weight_scale_inv"):
+                        continue
+                    weight_name = name.removesuffix("_scale_inv")
+                    if weight_name not in self.model_tensors:
+                        tensors_to_remove.append(name)
+                        continue
+                    w = self.model_tensors[weight_name]
+                    s = self.model_tensors[name]
+                    if w().dtype != torch.float8_e4m3fn:
+                        continue
+                    w_shape, s_shape = w().shape, s().shape
+                    block_size = None
+                    if prod(s_shape) != 1:
+                        if len(s_shape) != len(w_shape) or any(ws % ss for ws, ss in zip(w_shape, s_shape)):
+                            raise ValueError(f"FP8 weight {weight_name!r} {list(w_shape)} is not tiled by its scale {list(s_shape)}")
+                        block_size = [ws // ss for ws, ss in zip(w_shape, s_shape)]
+                    self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
+                    tensors_to_remove.append(name)
+                    if self._fp8_as_q8:
+                        self._fp8_dequantized.add(weight_name)
             elif quant_method is not None:
                 raise NotImplementedError(f"Quant method is not yet supported: {quant_method!r}")

diff --git a/conversion/qwen4exp.py b/conversion/qwen4exp.py
index 9680d2937..fe49cc03d 100644
--- a/conversion/qwen4exp.py
+++ b/conversion/qwen4exp.py
@@ -8,7 +8,7 @@ from torch import Tensor
 import gguf
 import numpy as np

-from .base import ModelBase
+from .base import LazyTorchTensor, ModelBase
 from .qwen import _LinearAttentionVReorderBase, _Qwen35MRopeMixin
 from .qwen3vl import Qwen3VLVisionModel

@@ -41,6 +41,18 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
         self._ple_shards: dict[int, str] = {}
         self._ple_row_dim: int | None = None
         self._mtp_fc: dict[str, Tensor] = {}
+        self._ple_scale: float | None = None
+
+    def dequant_model(self):
+        # FP8 checkpoints: all n-gram shards share one scale, stored under the parent module name
+        # there is no "ngram_embedding.weight", so the generic FP8 pass drops it or cannot map it
+        for name in list(self.model_tensors):
+            if name.endswith(".ngram_embedding.weight_scale"):
+                scale = LazyTorchTensor.to_eager(self.model_tensors.pop(name)()).float().flatten()
+                if scale.numel() != 1:
+                    raise ValueError(f"PLE scale {name!r} must be a scalar, got {scale.numel()} values")
+                self._ple_scale = float(scale[0])
+        super().dequant_model()

     @classmethod
     def filter_tensors(cls, item):
@@ -203,21 +215,31 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
                     f"PLE shard {shard} has row dim {int(shape[-1])}, expected {self._ple_row_dim}")
             rows += int(shape[0])

+        # an FP8 table keeps its bytes and gets a scale tensor, any other type has the scale folded in
+        keep_fp8 = name in self._fp8_e4m3_preserved
         table = gguf.LazyChunkedTensor(
-            [self._load_ple_shard(shard) for shard in shards],
+            [self._load_ple_shard(shard, keep_fp8) for shard in shards],
             shape=(rows, self._ple_row_dim),
-            dtype=np.float32,
+            dtype=np.uint8 if keep_fp8 else np.float32,
         )
         gguf_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD]
-        return [(gguf_name + ".weight", cast(Tensor, table))]
+        out = [(gguf_name + ".weight", cast(Tensor, table))]
+        if keep_fp8 and self._ple_scale is not None:
+            out.append((gguf_name + ".scale", torch.tensor([self._ple_scale], dtype=torch.float32)))
+        return out

-    def _load_ple_shard(self, name: str):
+    def _load_ple_shard(self, name: str, keep_fp8: bool):
         def load() -> np.ndarray:
-            from .base import LazyTorchTensor
-
             # a fresh lazy tensor every call, or to_eager() memoizes every shard
             eager = LazyTorchTensor.to_eager(self.model_tensors[name]())
-            return eager.to(torch.float32).contiguous().numpy()
+            if keep_fp8:
+                if eager.dtype != torch.float8_e4m3fn:
+                    raise ValueError(f"PLE shard {name} is {eager.dtype}, expected FP8 like the other shards")
+                return eager.contiguous().view(torch.uint8).numpy()
+            data = eager.to(torch.float32)
+            if self._ple_scale is not None:
+                data = data * self._ple_scale
+            return data.contiguous().numpy()
         return load

     def prepare_tensors(self):
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 4e9e23d10..338baa8f7 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -571,6 +571,7 @@ void llama_model_saver::add_tensors_from_model() {
     add_tensor(model->hc_head_base);
     add_tensor(model->hc_head_scale);
     add_tensor(model->per_layer_tok_embd);
+    add_tensor(model->per_layer_tok_embd_s);
     add_tensor(model->per_layer_model_proj);
     add_tensor(model->per_layer_proj_norm);
     add_tensor(model->value_embd);
diff --git a/src/llama-model.h b/src/llama-model.h
index 19b4c4288..f3a665410 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -881,6 +881,7 @@ struct llama_model {
     struct ggml_tensor * altup_proj           = nullptr;
     struct ggml_tensor * altup_unembd_proj    = nullptr;
     struct ggml_tensor * per_layer_tok_embd   = nullptr;
+    struct ggml_tensor * per_layer_tok_embd_s = nullptr; // optional scalar scale of an FP8 table

     // limite value embeddings
     struct ggml_tensor * value_embd = nullptr;
diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp
index 181e4dca4..faa4a928f 100644
--- a/src/models/qwen4exp.cpp
+++ b/src/models/qwen4exp.cpp
@@ -215,6 +215,8 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {

         per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"),
                                            { hparams.ple_head_dim, ple_rows }, TENSOR_READ_LAZY);
+        // an FP8 table keeps its checkpoint bytes, so its per-tensor scale is applied after the gather
+        per_layer_tok_embd_s = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "scale"), { 1 }, TENSOR_NOT_REQUIRED);
     }

     auto load_block = [&](int il, int flags) {
@@ -1408,6 +1410,9 @@ ggml_tensor * llama_model_qwen4exp::graph::build_inp_ple(

     // gather then flatten the heads: get_rows lays the head dimension out slowest, as the reference does
     ggml_tensor * emb = ggml_get_rows(ctx0, model.per_layer_tok_embd, rows);
+    if (model.per_layer_tok_embd_s) {
+        emb = ggml_mul(ctx0, emb, model.per_layer_tok_embd_s);
+    }
     emb = ggml_reshape_2d(ctx0, emb, hparams.ple_head_dim * n_heads, n_tokens);
     cb(emb, "ple_embd", -1);

diff --git a/tests/test_conversion_fp8.py b/tests/test_conversion_fp8.py
index 96387ad88..e191c8a55 100644
--- a/tests/test_conversion_fp8.py
+++ b/tests/test_conversion_fp8.py
@@ -25,6 +25,7 @@ from conversion.llama import LlamaModel
 from conversion.nemotron import NemotronHModel
 from conversion.phi import PhiMoeModel
 from conversion.qwen import Qwen2MoeModel, Qwen3NextModel
+from conversion.qwen4exp import Qwen4ExpTextModel


 class ForcedF16Model(Qwen2MoeModel):
@@ -366,6 +367,27 @@ class TestFP8Conversion(unittest.TestCase):
         self.assertEqual(dtype, gguf.GGMLQuantizationType.Q8_0)
         np.testing.assert_allclose(gguf.dequantize(data, dtype), np.full((4, 32), 0.25), atol=1e-4)

+    def test_modelopt_block_scale_inv(self):
+        name = "model.layers.0.self_attn.q_proj.weight"
+        weight = (torch.arange(32, dtype=torch.float32).reshape(4, 8) - 16).to(torch.float8_e4m3fn)
+        scale = torch.tensor([[0.25, 0.5], [1.0, 2.0]])
+        expected = weight.float() * scale.repeat_interleave(2, 0).repeat_interleave(4, 1)
+        for eager in (False, True):
+            with self.subTest(eager=eager):
+                result = self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": scale}, eager=eager, check_lazy=not eager)
+                self.assertEqual(set(result), {"blk.0.attn_q.weight"})
+                dtype, data = result["blk.0.attn_q.weight"]
+                self.assertEqual(dtype, gguf.GGMLQuantizationType.F16)
+                np.testing.assert_array_equal(data, expected.numpy())
+
+        with self.assertRaisesRegex(ValueError, "not tiled"):
+            self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": torch.ones((3, 2))})
+
+        # a scalar scale keeps the FP8 bytes, as with weight_scale
+        result = self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": torch.tensor([0.25])}, check_lazy=True)
+        self.assertEqual(result["blk.0.attn_q.weight"][0], gguf.GGMLQuantizationType.F8_E4M3)
+        np.testing.assert_array_equal(result["blk.0.attn_q.scale"][1], [0.25])
+

 class TestInkling2PackedConversion(unittest.TestCase):
     """Conversion of the packed Inkling2 expert weights (see _dequant_expert_chunk in conversion/inkling2.py).
@@ -675,5 +697,84 @@ class TestInkling2TextNames(unittest.TestCase):
             np.testing.assert_array_equal(renamed[name], actual[name], name)


+class TestQwen4ExpPleScale(unittest.TestCase):
+    """FP8 checkpoints store the n-gram table as shards with one shared scale under the parent module name."""
+
+    PREFIX = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding."
+    SHARDS = (torch.tensor([[1., -2.], [3., 4.]]), torch.tensor([[5., 6.], [-7., 8.]]))
+    TABLE = "per_layer_token_embd.weight"
+    SCALE = "per_layer_token_embd.scale"
+
+    def convert(self, ftype=gguf.LlamaFileType.GUESSED, scaled=True, quant_method="modelopt", **kwargs):
+        # shard 1 first: the row order must follow the shard index
+        tensors = {self.PREFIX + f"shard_{i}.weight": self.SHARDS[i].to(torch.float8_e4m3fn) for i in (1, 0)}
+        if scaled:
+            tensors[self.PREFIX + "weight_scale"] = torch.tensor([0.25], dtype=torch.bfloat16)
+        hparams = {
+            "model_type": "qwen4exp",
+            "architectures": ["Qwen4ExpForCausalLM"],
+            "num_hidden_layers": 2,
+            "mtp_num_hidden_layers": 1,
+            "split_ngram_parts": 2,
+            "quantization_config": {"quant_method": quant_method},
+        }
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory)
+            (path / "config.json").write_text(json.dumps(hparams))
+            save_file(tensors, str(path / "model.safetensors"))
+            model = Qwen4ExpTextModel(path, ftype, path / "out.gguf", hparams=hparams, **kwargs)
+            model.prepare_tensors()
+            result = {}
+            for name, info in model.gguf_writer.tensors[0].items():
+                if isinstance(info.tensor, gguf.LazyChunkedTensor):
+                    # written like the GGUF writer does, a chunked tensor has no other read path
+                    with (path / "tensor.bin").open("wb") as f:
+                        info.tensor.tofile(f)
+                    data = np.frombuffer((path / "tensor.bin").read_bytes(), dtype=info.tensor.dtype).reshape(info.tensor.shape)
+                else:
+                    data = np.asarray(gguf.LazyNumpyTensor.to_eager(info.tensor))
+                result[name] = (info.dtype, data)
+            return result
+
+    def test_fp8_table_keeps_bytes_and_scale(self):
+        raw = torch.cat(self.SHARDS).to(torch.float8_e4m3fn).view(torch.uint8).numpy()
+        for kwargs, names in (
+            ({}, {self.TABLE, self.SCALE}),
+            ({"quant_method": "fp8"}, {self.TABLE, self.SCALE}),
+            ({"ple_only": True}, {self.TABLE}),
+            ({"no_ple": True}, {self.SCALE}),
+        ):
+            with self.subTest(**kwargs):
+                result = self.convert(**kwargs)
+                self.assertEqual(set(result), names)
+                if self.TABLE in result:
+                    dtype, data = result[self.TABLE]
+                    self.assertEqual(dtype, gguf.GGMLQuantizationType.F8_E4M3)
+                    np.testing.assert_array_equal(data, raw)
+                if self.SCALE in result:
+                    dtype, data = result[self.SCALE]
+                    self.assertEqual(dtype, gguf.GGMLQuantizationType.F32)
+                    np.testing.assert_array_equal(data, [0.25])
+
+        result = self.convert(scaled=False)
+        self.assertEqual(set(result), {self.TABLE})
+        np.testing.assert_array_equal(result[self.TABLE][1], raw)
+
+    def test_scale_is_folded_into_other_types(self):
+        values = torch.cat(self.SHARDS).numpy()
+        for kwargs, qtype in (
+            ({"ftype": gguf.LlamaFileType.ALL_F32}, gguf.GGMLQuantizationType.F32),
+            ({"ftype": gguf.LlamaFileType.MOSTLY_F16}, gguf.GGMLQuantizationType.F16),
+            ({"fp8_as_q8": True}, gguf.GGMLQuantizationType.F16),
+        ):
+            for scaled in (False, True):
+                with self.subTest(scaled=scaled, **kwargs):
+                    result = self.convert(scaled=scaled, **kwargs)
+                    self.assertEqual(set(result), {self.TABLE})
+                    dtype, data = result[self.TABLE]
+                    self.assertEqual(dtype, qtype)
+                    np.testing.assert_array_equal(data, values * (0.25 if scaled else 1.))
+
+
 if __name__ == "__main__":
     unittest.main()