Commit b84ea5cf5 for llama.cpp
commit b84ea5cf5e56dd934e0d52c6ffdfffbf06f58d2f
Author: Aaron Teo <aaron.teo1@ibm.com>
Date: Fri Oct 16 16:18:29 2026 +0800
convert : speed up model endian conversion and add more datatype support (#30647)
* convert: speed up model endian conversion
Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
* convert: explicitly state noop
Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
* convert: add missing data type support
Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
---------
Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
diff --git a/gguf-py/gguf/scripts/gguf_convert_endian.py b/gguf-py/gguf/scripts/gguf_convert_endian.py
index 31618acfc..3b20998c6 100755
--- a/gguf-py/gguf/scripts/gguf_convert_endian.py
+++ b/gguf-py/gguf/scripts/gguf_convert_endian.py
@@ -19,71 +19,27 @@ import gguf
logger = logging.getLogger("gguf-convert-endian")
-def byteswap_noop(tensor, block_offs):
- # this function is used when byteswapping is not needed
- pass
-
-
-def byteswap_q4_0(tensor, block_offs):
- # Each block_q4_0 consists of an f16 delta (scaling factor) followed by 16 int8 quantizations.
-
- # Byte-Swap f16 sized delta field
- delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)
- delta.byteswap(inplace=True)
-
-
-def byteswap_q8_0(tensor, block_offs):
- # Each block_q8_0 consists of an f16 delta (scaling factor) followed by 32 int8 quantizations.
-
- # Byte-Swap f16 sized delta field
- delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)
- delta.byteswap(inplace=True)
-
-
-def byteswap_q4_k(tensor, block_offs):
- # Each block_q4_k consists of 2 f16 values followed by 140 int8 values.
-
- # Byte-Swap f16 sized fields
- delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)
- delta.byteswap(inplace=True)
-
- delta = tensor.data[block_offs + 2:block_offs + 4].view(dtype=np.uint16)
- delta.byteswap(inplace=True)
-
-
-def byteswap_q6_k(tensor, block_offs):
- # Each block_q6_k consists of 208 int8 values followed by 1 f16 value.
-
- # Byte-Swap f16 sized field
- delta = tensor.data[block_offs + 208:block_offs + 210].view(dtype=np.uint16)
- delta.byteswap(inplace=True)
-
-
-def byteswap_q1_0(tensor, block_offs):
- # Each block_q1_0 consists of an f16 delta followed by 16 int8 quantizations.
-
- # Byte-Swap f16 sized delta field
- delta = tensor.data[block_offs:block_offs + 2].view(dtype=np.uint16)
- delta.byteswap(inplace=True)
-
-
-def byteswap_tq2_0(tensor, block_offs):
- # Each block_tq2_0 consists of 64 int8 values followed by 1 f16 value.
-
- # Byte-Swap f16 sized field
- delta = tensor.data[block_offs + 64:block_offs + 66].view(dtype=np.uint16)
- delta.byteswap(inplace=True)
-
-
-byteswap_tensors = {
- gguf.GGMLQuantizationType.Q1_0: byteswap_q1_0,
- gguf.GGMLQuantizationType.Q4_0: byteswap_q4_0,
- gguf.GGMLQuantizationType.Q8_0: byteswap_q8_0,
- gguf.GGMLQuantizationType.Q4_K: byteswap_q4_k,
- gguf.GGMLQuantizationType.Q6_K: byteswap_q6_k,
- gguf.GGMLQuantizationType.TQ2_0: byteswap_tq2_0,
- gguf.GGMLQuantizationType.MXFP4: byteswap_noop,
- gguf.GGMLQuantizationType.NVFP4: byteswap_noop,
+# byte offset and type of the multi-byte fields in each block, all other fields are single bytes
+# use the width ggml reads the field with, e.g. Q5_0 qh is uint8_t[4] but read as uint32_t
+byteswap_tensors: dict[gguf.GGMLQuantizationType, tuple[tuple[int, type[np.unsignedinteger]], ...]] = {
+ gguf.GGMLQuantizationType.Q1_0: ((0, np.uint16),), # d
+ gguf.GGMLQuantizationType.Q2_0: ((0, np.uint16),), # d
+ gguf.GGMLQuantizationType.Q4_0: ((0, np.uint16),), # d
+ gguf.GGMLQuantizationType.Q4_1: ((0, np.uint16), (2, np.uint16)), # d, m
+ gguf.GGMLQuantizationType.Q5_0: ((0, np.uint16), (2, np.uint32)), # d, qh
+ gguf.GGMLQuantizationType.Q5_1: ((0, np.uint16), (2, np.uint16), (4, np.uint32)), # d, m, qh
+ gguf.GGMLQuantizationType.Q8_0: ((0, np.uint16),), # d
+ gguf.GGMLQuantizationType.Q2_K: ((80, np.uint16), (82, np.uint16)), # d, dmin
+ gguf.GGMLQuantizationType.Q3_K: ((108, np.uint16),), # d
+ gguf.GGMLQuantizationType.Q4_K: ((0, np.uint16), (2, np.uint16)), # d, dmin
+ gguf.GGMLQuantizationType.Q5_K: ((0, np.uint16), (2, np.uint16)), # d, dmin
+ gguf.GGMLQuantizationType.Q6_K: ((208, np.uint16),), # d
+ gguf.GGMLQuantizationType.IQ4_NL: ((0, np.uint16),), # d
+ gguf.GGMLQuantizationType.IQ4_XS: ((0, np.uint16), (2, np.uint16)), # d, scales_h
+ gguf.GGMLQuantizationType.TQ1_0: ((52, np.uint16),), # d
+ gguf.GGMLQuantizationType.TQ2_0: ((64, np.uint16),), # d
+ gguf.GGMLQuantizationType.MXFP4: (), # noop
+ gguf.GGMLQuantizationType.NVFP4: (), # noop
}
@@ -140,28 +96,12 @@ def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> None
# Byte-swap tensor data if necessary
if tensor.tensor_type in byteswap_tensors:
- # first flatten structure
- oldshape = tensor.data.shape
- newshape = 1
- for i in tensor.data.shape:
- newshape *= i
-
- tensor.data.resize(newshape)
-
- block_size = gguf.constants.GGML_QUANT_SIZES[tensor.tensor_type][1]
- byteswap_func = byteswap_tensors[tensor.tensor_type]
-
- n_blocks = len(tensor.data) // block_size
- for block_num in (inner_pbar := tqdm(range(n_blocks), desc="Byte-swapping Blocks", leave=False)):
- block_offs = block_num * block_size
-
- byteswap_func(tensor, block_offs)
-
- if block_num % 100000 == 0:
- inner_pbar.set_description(f"Byte-swapping Blocks [{(n_blocks - block_num) // n_blocks}]")
-
- # restore old shape in case it's ever used
- tensor.data.resize(oldshape)
+ # one row per block, then swap each field of all blocks at once
+ type_size = gguf.constants.GGML_QUANT_SIZES[tensor.tensor_type][1]
+ blocks = tensor.data.reshape(-1, type_size, copy=False)
+ for offs, dtype in byteswap_tensors[tensor.tensor_type]:
+ size = np.dtype(dtype).itemsize
+ blocks[:, offs:offs + size].view(dtype=dtype)[:, 0].byteswap(inplace=True)
elif tensor.tensor_type == gguf.GGMLQuantizationType.BF16:
# Special case for BF16
# It is 2-bytes data, but by default view loads it as 1-byte data.
@@ -173,6 +113,7 @@ def convert_byteorder(reader: gguf.GGUFReader, args: argparse.Namespace) -> None
pbar.set_description(log_message)
+ reader.data.flush()
logger.info("* Completion")