Commit d3064fddc for llama.cpp
commit d3064fddce83046ef66eded524cf00947d5166f3
Author: Aman Gupta <amangupta052@gmail.com>
Date: Tue Nov 17 22:25:01 2026 +0800
convert : fix FP8 qwen4exp checkpoints (n-gram table scale, block-scaled FP8) (#32476)
* convert : fix FP8 n-gram table export for qwen4exp
* convert : dequantize block-scaled FP8 weight_scale_inv in ModelOpt checkpoints
diff --git a/conversion/base.py b/conversion/base.py
index a95ecc008..7cfbff321 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -653,6 +653,28 @@ class ModelBase:
raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported")
elif quant_method in ("modelopt", "modelopt_mixed"):
dequant_fp8()
+ # FP8_PB_WO: one scale per block, the block size follows from the two shapes
+ for name in self.model_tensors.keys():
+ if not name.endswith(".weight_scale_inv"):
+ continue
+ weight_name = name.removesuffix("_scale_inv")
+ if weight_name not in self.model_tensors:
+ tensors_to_remove.append(name)
+ continue
+ w = self.model_tensors[weight_name]
+ s = self.model_tensors[name]
+ if w().dtype != torch.float8_e4m3fn:
+ continue
+ w_shape, s_shape = w().shape, s().shape
+ block_size = None
+ if prod(s_shape) != 1:
+ if len(s_shape) != len(w_shape) or any(ws % ss for ws, ss in zip(w_shape, s_shape)):
+ raise ValueError(f"FP8 weight {weight_name!r} {list(w_shape)} is not tiled by its scale {list(s_shape)}")
+ block_size = [ws // ss for ws, ss in zip(w_shape, s_shape)]
+ self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
+ tensors_to_remove.append(name)
+ if self._fp8_as_q8:
+ self._fp8_dequantized.add(weight_name)
elif quant_method is not None:
raise NotImplementedError(f"Quant method is not yet supported: {quant_method!r}")
diff --git a/conversion/qwen4exp.py b/conversion/qwen4exp.py
index 9680d2937..fe49cc03d 100644
--- a/conversion/qwen4exp.py
+++ b/conversion/qwen4exp.py
@@ -8,7 +8,7 @@ from torch import Tensor
import gguf
import numpy as np
-from .base import ModelBase
+from .base import LazyTorchTensor, ModelBase
from .qwen import _LinearAttentionVReorderBase, _Qwen35MRopeMixin
from .qwen3vl import Qwen3VLVisionModel
@@ -41,6 +41,18 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
self._ple_shards: dict[int, str] = {}
self._ple_row_dim: int | None = None
self._mtp_fc: dict[str, Tensor] = {}
+ self._ple_scale: float | None = None
+
+ def dequant_model(self):
+ # FP8 checkpoints: all n-gram shards share one scale, stored under the parent module name
+ # there is no "ngram_embedding.weight", so the generic FP8 pass drops it or cannot map it
+ for name in list(self.model_tensors):
+ if name.endswith(".ngram_embedding.weight_scale"):
+ scale = LazyTorchTensor.to_eager(self.model_tensors.pop(name)()).float().flatten()
+ if scale.numel() != 1:
+ raise ValueError(f"PLE scale {name!r} must be a scalar, got {scale.numel()} values")
+ self._ple_scale = float(scale[0])
+ super().dequant_model()
@classmethod
def filter_tensors(cls, item):
@@ -203,21 +215,31 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
f"PLE shard {shard} has row dim {int(shape[-1])}, expected {self._ple_row_dim}")
rows += int(shape[0])
+ # an FP8 table keeps its bytes and gets a scale tensor, any other type has the scale folded in
+ keep_fp8 = name in self._fp8_e4m3_preserved
table = gguf.LazyChunkedTensor(
- [self._load_ple_shard(shard) for shard in shards],
+ [self._load_ple_shard(shard, keep_fp8) for shard in shards],
shape=(rows, self._ple_row_dim),
- dtype=np.float32,
+ dtype=np.uint8 if keep_fp8 else np.float32,
)
gguf_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD]
- return [(gguf_name + ".weight", cast(Tensor, table))]
+ out = [(gguf_name + ".weight", cast(Tensor, table))]
+ if keep_fp8 and self._ple_scale is not None:
+ out.append((gguf_name + ".scale", torch.tensor([self._ple_scale], dtype=torch.float32)))
+ return out
- def _load_ple_shard(self, name: str):
+ def _load_ple_shard(self, name: str, keep_fp8: bool):
def load() -> np.ndarray:
- from .base import LazyTorchTensor
-
# a fresh lazy tensor every call, or to_eager() memoizes every shard
eager = LazyTorchTensor.to_eager(self.model_tensors[name]())
- return eager.to(torch.float32).contiguous().numpy()
+ if keep_fp8:
+ if eager.dtype != torch.float8_e4m3fn:
+ raise ValueError(f"PLE shard {name} is {eager.dtype}, expected FP8 like the other shards")
+ return eager.contiguous().view(torch.uint8).numpy()
+ data = eager.to(torch.float32)
+ if self._ple_scale is not None:
+ data = data * self._ple_scale
+ return data.contiguous().numpy()
return load
def prepare_tensors(self):
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 4e9e23d10..338baa8f7 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -571,6 +571,7 @@ void llama_model_saver::add_tensors_from_model() {
add_tensor(model->hc_head_base);
add_tensor(model->hc_head_scale);
add_tensor(model->per_layer_tok_embd);
+ add_tensor(model->per_layer_tok_embd_s);
add_tensor(model->per_layer_model_proj);
add_tensor(model->per_layer_proj_norm);
add_tensor(model->value_embd);
diff --git a/src/llama-model.h b/src/llama-model.h
index 19b4c4288..f3a665410 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -881,6 +881,7 @@ struct llama_model {
struct ggml_tensor * altup_proj = nullptr;
struct ggml_tensor * altup_unembd_proj = nullptr;
struct ggml_tensor * per_layer_tok_embd = nullptr;
+ struct ggml_tensor * per_layer_tok_embd_s = nullptr; // optional scalar scale of an FP8 table
// limite value embeddings
struct ggml_tensor * value_embd = nullptr;
diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp
index 181e4dca4..faa4a928f 100644
--- a/src/models/qwen4exp.cpp
+++ b/src/models/qwen4exp.cpp
@@ -215,6 +215,8 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"),
{ hparams.ple_head_dim, ple_rows }, TENSOR_READ_LAZY);
+ // an FP8 table keeps its checkpoint bytes, so its per-tensor scale is applied after the gather
+ per_layer_tok_embd_s = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "scale"), { 1 }, TENSOR_NOT_REQUIRED);
}
auto load_block = [&](int il, int flags) {
@@ -1408,6 +1410,9 @@ ggml_tensor * llama_model_qwen4exp::graph::build_inp_ple(
// gather then flatten the heads: get_rows lays the head dimension out slowest, as the reference does
ggml_tensor * emb = ggml_get_rows(ctx0, model.per_layer_tok_embd, rows);
+ if (model.per_layer_tok_embd_s) {
+ emb = ggml_mul(ctx0, emb, model.per_layer_tok_embd_s);
+ }
emb = ggml_reshape_2d(ctx0, emb, hparams.ple_head_dim * n_heads, n_tokens);
cb(emb, "ple_embd", -1);
diff --git a/tests/test_conversion_fp8.py b/tests/test_conversion_fp8.py
index 96387ad88..e191c8a55 100644
--- a/tests/test_conversion_fp8.py
+++ b/tests/test_conversion_fp8.py
@@ -25,6 +25,7 @@ from conversion.llama import LlamaModel
from conversion.nemotron import NemotronHModel
from conversion.phi import PhiMoeModel
from conversion.qwen import Qwen2MoeModel, Qwen3NextModel
+from conversion.qwen4exp import Qwen4ExpTextModel
class ForcedF16Model(Qwen2MoeModel):
@@ -366,6 +367,27 @@ class TestFP8Conversion(unittest.TestCase):
self.assertEqual(dtype, gguf.GGMLQuantizationType.Q8_0)
np.testing.assert_allclose(gguf.dequantize(data, dtype), np.full((4, 32), 0.25), atol=1e-4)
+ def test_modelopt_block_scale_inv(self):
+ name = "model.layers.0.self_attn.q_proj.weight"
+ weight = (torch.arange(32, dtype=torch.float32).reshape(4, 8) - 16).to(torch.float8_e4m3fn)
+ scale = torch.tensor([[0.25, 0.5], [1.0, 2.0]])
+ expected = weight.float() * scale.repeat_interleave(2, 0).repeat_interleave(4, 1)
+ for eager in (False, True):
+ with self.subTest(eager=eager):
+ result = self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": scale}, eager=eager, check_lazy=not eager)
+ self.assertEqual(set(result), {"blk.0.attn_q.weight"})
+ dtype, data = result["blk.0.attn_q.weight"]
+ self.assertEqual(dtype, gguf.GGMLQuantizationType.F16)
+ np.testing.assert_array_equal(data, expected.numpy())
+
+ with self.assertRaisesRegex(ValueError, "not tiled"):
+ self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": torch.ones((3, 2))})
+
+ # a scalar scale keeps the FP8 bytes, as with weight_scale
+ result = self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": torch.tensor([0.25])}, check_lazy=True)
+ self.assertEqual(result["blk.0.attn_q.weight"][0], gguf.GGMLQuantizationType.F8_E4M3)
+ np.testing.assert_array_equal(result["blk.0.attn_q.scale"][1], [0.25])
+
class TestInkling2PackedConversion(unittest.TestCase):
"""Conversion of the packed Inkling2 expert weights (see _dequant_expert_chunk in conversion/inkling2.py).
@@ -675,5 +697,84 @@ class TestInkling2TextNames(unittest.TestCase):
np.testing.assert_array_equal(renamed[name], actual[name], name)
+class TestQwen4ExpPleScale(unittest.TestCase):
+ """FP8 checkpoints store the n-gram table as shards with one shared scale under the parent module name."""
+
+ PREFIX = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding."
+ SHARDS = (torch.tensor([[1., -2.], [3., 4.]]), torch.tensor([[5., 6.], [-7., 8.]]))
+ TABLE = "per_layer_token_embd.weight"
+ SCALE = "per_layer_token_embd.scale"
+
+ def convert(self, ftype=gguf.LlamaFileType.GUESSED, scaled=True, quant_method="modelopt", **kwargs):
+ # shard 1 first: the row order must follow the shard index
+ tensors = {self.PREFIX + f"shard_{i}.weight": self.SHARDS[i].to(torch.float8_e4m3fn) for i in (1, 0)}
+ if scaled:
+ tensors[self.PREFIX + "weight_scale"] = torch.tensor([0.25], dtype=torch.bfloat16)
+ hparams = {
+ "model_type": "qwen4exp",
+ "architectures": ["Qwen4ExpForCausalLM"],
+ "num_hidden_layers": 2,
+ "mtp_num_hidden_layers": 1,
+ "split_ngram_parts": 2,
+ "quantization_config": {"quant_method": quant_method},
+ }
+ with tempfile.TemporaryDirectory() as directory:
+ path = Path(directory)
+ (path / "config.json").write_text(json.dumps(hparams))
+ save_file(tensors, str(path / "model.safetensors"))
+ model = Qwen4ExpTextModel(path, ftype, path / "out.gguf", hparams=hparams, **kwargs)
+ model.prepare_tensors()
+ result = {}
+ for name, info in model.gguf_writer.tensors[0].items():
+ if isinstance(info.tensor, gguf.LazyChunkedTensor):
+ # written like the GGUF writer does, a chunked tensor has no other read path
+ with (path / "tensor.bin").open("wb") as f:
+ info.tensor.tofile(f)
+ data = np.frombuffer((path / "tensor.bin").read_bytes(), dtype=info.tensor.dtype).reshape(info.tensor.shape)
+ else:
+ data = np.asarray(gguf.LazyNumpyTensor.to_eager(info.tensor))
+ result[name] = (info.dtype, data)
+ return result
+
+ def test_fp8_table_keeps_bytes_and_scale(self):
+ raw = torch.cat(self.SHARDS).to(torch.float8_e4m3fn).view(torch.uint8).numpy()
+ for kwargs, names in (
+ ({}, {self.TABLE, self.SCALE}),
+ ({"quant_method": "fp8"}, {self.TABLE, self.SCALE}),
+ ({"ple_only": True}, {self.TABLE}),
+ ({"no_ple": True}, {self.SCALE}),
+ ):
+ with self.subTest(**kwargs):
+ result = self.convert(**kwargs)
+ self.assertEqual(set(result), names)
+ if self.TABLE in result:
+ dtype, data = result[self.TABLE]
+ self.assertEqual(dtype, gguf.GGMLQuantizationType.F8_E4M3)
+ np.testing.assert_array_equal(data, raw)
+ if self.SCALE in result:
+ dtype, data = result[self.SCALE]
+ self.assertEqual(dtype, gguf.GGMLQuantizationType.F32)
+ np.testing.assert_array_equal(data, [0.25])
+
+ result = self.convert(scaled=False)
+ self.assertEqual(set(result), {self.TABLE})
+ np.testing.assert_array_equal(result[self.TABLE][1], raw)
+
+ def test_scale_is_folded_into_other_types(self):
+ values = torch.cat(self.SHARDS).numpy()
+ for kwargs, qtype in (
+ ({"ftype": gguf.LlamaFileType.ALL_F32}, gguf.GGMLQuantizationType.F32),
+ ({"ftype": gguf.LlamaFileType.MOSTLY_F16}, gguf.GGMLQuantizationType.F16),
+ ({"fp8_as_q8": True}, gguf.GGMLQuantizationType.F16),
+ ):
+ for scaled in (False, True):
+ with self.subTest(scaled=scaled, **kwargs):
+ result = self.convert(scaled=scaled, **kwargs)
+ self.assertEqual(set(result), {self.TABLE})
+ dtype, data = result[self.TABLE]
+ self.assertEqual(dtype, qtype)
+ np.testing.assert_array_equal(data, values * (0.25 if scaled else 1.))
+
+
if __name__ == "__main__":
unittest.main()