Commit e937b17a2 for llama.cpp
commit e937b17a2df081e9c7ae380f0dab4341c1585bce
Author: Jhen-Jie Hong <iainst0409@gmail.com>
Date: Mon Oct 12 11:52:01 2026 +0800
hexagon: optimize q2_k bpw and activation (#30035)
diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
index b214b1543..533f51a0e 100644
--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp
+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp
@@ -288,8 +288,8 @@ static inline bool ggml_hexagon_is_repack_type(enum ggml_type type) {
type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q2_K;
}
-// Size of one repacked row in the DSP tiled layout. The K-quant tiles store uncompressed scales/mins,
-// so they are larger than the ggml blocks. For the other repack types the tile has the same size as the ggml blocks.
+// Size of one repacked row in the DSP tiled layout. The Q3_K..Q6_K tiles store uncompressed scales/mins,
+// so they are larger than the ggml blocks. For the other repack types (Q2_K included) the tile has the same size.
static inline size_t ggml_hexagon_tiled_row_size(enum ggml_type type, int64_t ne0) {
if (type == GGML_TYPE_Q6_K) {
return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q6_K / 32);
@@ -303,9 +303,6 @@ static inline size_t ggml_hexagon_tiled_row_size(enum ggml_type type, int64_t ne
if (type == GGML_TYPE_Q3_K) {
return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q3_K / 32);
}
- if (type == GGML_TYPE_Q2_K) {
- return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q2_K / 32);
- }
return ggml_row_size(type, ne0);
}
@@ -1909,13 +1906,11 @@ static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset,
for (int kt = 0; kt < n_k_tiles; kt++) {
const int kt_local = kt % 8; // k-tile within the super-block
const block_q2_K * b = &src_row[kt / 8];
- const float d = GGML_FP16_TO_FP32(b->d);
- const float dmin = GGML_FP16_TO_FP32(b->dmin);
uint8_t * tile = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size;
uint8_t * lo_pl = tile;
- ggml_half * sc_pl = (ggml_half *) (tile + 256);
- ggml_half * m_pl = (ggml_half *) (tile + 384);
+ uint8_t * sc_pl = tile + 256;
+ ggml_half * dd_pl = (ggml_half *) (tile + 320);
for (int lk = 0; lk < 32; lk++) {
const int g = lk >> 2;
@@ -1923,9 +1918,10 @@ static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset,
lo_pl[(g >> 2) * 128 + pos] |= (uint8_t) (q2_3_K_get_low2(b->qs, kt_local * 32 + lk) << ((g & 3) * 2));
}
for (int sub = 0; sub < 2; sub++) {
- const uint8_t sc = b->scales[kt_local * 2 + sub];
- sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16( d * (float) (sc & 0xF));
- m_pl [sub * 32 + row] = GGML_FP32_TO_FP16(-dmin * (float) (sc >> 4));
+ sc_pl[sub * 32 + row] = b->scales[kt_local * 2 + sub];
+ }
+ if (row / 8 == kt_local % 4) {
+ dd_pl[row % 8] = kt_local < 4 ? b->d : (ggml_half) (b->dmin ^ 0x8000); // -dmin
}
}
}
@@ -1935,7 +1931,7 @@ static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset,
GGML_UNUSED(size);
}
-// Reverse of repack_q2_K_tiled. Unpacks quants losslessly and normalizes scales/mins. Read-back only.
+// Reverse of repack_q2_K_tiled. Lossless. Read-back only.
static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset, size_t size) {
GGML_ASSERT(offset == 0);
@@ -1970,14 +1966,12 @@ static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset,
block_q2_K * b = &dst_row[sb];
memset(b, 0, sizeof(block_q2_K));
- ggml_half sub_scales[16];
- ggml_half sub_mins[16];
for (int kt_local = 0; kt_local < 8; kt_local++) {
const int kt = sb * 8 + kt_local;
const uint8_t * tile = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size;
const uint8_t * lo_pl = tile;
- const ggml_half * sc_pl = (const ggml_half *) (tile + 256);
- const ggml_half * m_pl = (const ggml_half *) (tile + 384);
+ const uint8_t * sc_pl = tile + 256;
+ const ggml_half * dd_pl = (const ggml_half *) (tile + 320);
for (int lk = 0; lk < 32; lk++) {
const int e = kt_local * 32 + lk;
@@ -1988,37 +1982,16 @@ static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset,
}
for (int sub = 0; sub < 2; sub++) {
- const float D = GGML_FP16_TO_FP32(sc_pl[sub * 32 + row]);
- const float M = GGML_FP16_TO_FP32(m_pl[sub * 32 + row]);
- sub_scales[kt_local * 2 + sub] = GGML_FP32_TO_FP16((D > 0.0f) ? D : 0.0f);
- sub_mins[kt_local * 2 + sub] = GGML_FP32_TO_FP16((-M > 0.0f) ? -M : 0.0f);
- }
- }
-
- int ls[16];
- int lm[16];
- ggml_half * const dd[2] = { &b->d, &b->dmin };
- const ggml_half * const prod[2] = { sub_scales, sub_mins };
- int * const ll[2] = { ls, lm };
- for (int w = 0; w < 2; w++) {
- if (hexagon_recover_k_scales(prod[w], 16, 0, 15, dd[w], ll[w])) {
- continue;
- }
- float max_val = 0.0f;
- for (int j = 0; j < 16; j++) {
- max_val = (std::max)(max_val, GGML_FP16_TO_FP32(prod[w][j]));
+ b->scales[kt_local * 2 + sub] = sc_pl[sub * 32 + row];
}
- *dd[w] = GGML_FP32_TO_FP16(max_val / 15.0f);
- const float d_actual = GGML_FP16_TO_FP32(*dd[w]);
- const float inv_d = (d_actual > 0.0f) ? (1.0f / d_actual) : 0.0f;
- for (int j = 0; j < 16; j++) {
- ll[w][j] = (std::min)(15, (int) roundf(inv_d * GGML_FP16_TO_FP32(prod[w][j])));
+ if (row / 8 == kt_local % 4) {
+ if (kt_local < 4) {
+ b->d = dd_pl[row % 8];
+ } else {
+ b->dmin = (ggml_half) (dd_pl[row % 8] ^ 0x8000);
+ }
}
}
-
- for (int j = 0; j < 16; j++) {
- b->scales[j] = (uint8_t) (ls[j] | (lm[j] << 4));
- }
}
}
}
diff --git a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
index 40404bf10..f0a0d9de1 100644
--- a/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
+++ b/ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
@@ -701,20 +701,29 @@ static void dequantize_tiled_weight_to_fp16_task_q3_k(
}
}
-// Q2_K stores 2-bit weights and one fp16 scale and offset per 16 k, see HTP_MM_WEIGHT_TILE_SIZE_Q2_K.
+// Q2_K stores 2-bit weights, the ggml scale bytes and d / -dmin per super-block, see HTP_MM_WEIGHT_TILE_SIZE_Q2_K.
static void dequantize_tiled_weight_to_fp16_task_q2_k(
const tiled_dequantize_state_t *state,
uint32_t start_tile, uint32_t end_tile) {
const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
+ HVX_Vector v_d = Q6_V_vzero();
+ HVX_Vector v_dm = Q6_V_vzero();
+
for (uint32_t t = start_tile; t < end_tile; t++) {
- const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size);
+ const uint8_t * tile = state->src + t * state->aligned_tile_size;
+ const HVX_Vector * vptr = (const HVX_Vector *) tile;
__fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS;
- HVX_Vector v_sc = vptr[2];
+ // rows hold all k tiles and n_k_tiles is a multiple of 8, so t % 8 is the tile within its super-block
+ if (t == start_tile || (t & 7) == 0) {
+ q2_k_load_dmin(tile - (t & 7) * state->aligned_tile_size, &v_d, &v_dm);
+ }
+
+ HVX_Vector v_sc, v_m;
+ q2_k_unpack_scales(tile, v_d, v_dm, &v_sc, &v_m);
HVX_Vector v_sc_k16 = Q6_V_vror_VR(v_sc, 64);
- HVX_Vector v_m = vptr[3];
HVX_Vector v_m_k16 = Q6_V_vror_VR(v_m, 64);
HVX_Vector v_scale_k0 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2));
HVX_Vector v_scale_k16 = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2));
diff --git a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
index 580bae4ac..0c86e407e 100644
--- a/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
+++ b/ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h
@@ -1,19 +1,31 @@
// Dynamic quantizers that produce tiled activations
-// vector 9: d * sum(q) of the 32 k, replicated (q8_1), or per 16 k for Q2_K (q8_1_s16): k 0..15 in lanes 0..31,
-// k 16..31 in lanes 32..63, with d the fp16 scale of vector 8
+// q8_1: vector 9 holds d * sum(q) of the 32 k, replicated.
+// q8_1_s16 (Q2_K, sums16): one fp16 d (vector 8) for the 256 k super-block starting at x_sb, and vector 9 holds the
+// integer sums of k 0..15 (low halfword) and k 16..31 (high halfword) in every word
__attribute__((always_inline))
-static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_t * restrict y_block, const bool sums16) {
+static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_t * restrict y_block, const bool sums16,
+ const float * restrict x_sb) {
assert((unsigned long) x % 128 == 0);
assert((unsigned long) y_block % 128 == 0);
HVX_Vector * vx = (HVX_Vector *) x;
HVX_Vector zero = Q6_V_vzero();
- HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0]));
- HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1]));
- HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2]));
- HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3]));
+ HVX_Vector vmax0_sf, vmax1_sf, vmax2_sf, vmax3_sf;
+ if (sums16) {
+ const HVX_Vector * vsb = (const HVX_Vector *) x_sb;
+ HVX_Vector vm = hvx_vec_abs_f32(vsb[0]);
+ for (int i = 1; i < 8; i++) {
+ vm = Q6_Vsf_vmax_VsfVsf(vm, hvx_vec_abs_f32(vsb[i]));
+ }
+ vmax0_sf = vmax1_sf = vmax2_sf = vmax3_sf = hvx_vec_reduce_max_f32(vm);
+ } else {
+ vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0]));
+ vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1]));
+ vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2]));
+ vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3]));
+ }
HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero);
HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero);
@@ -61,15 +73,6 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
HVX_Vector vd3_sf = hvx_vec_mul_f32_f32(vmax3_sf, v_inv127);
HVX_Vector v_sums_sf = Q6_Vsf_equals_Vw(v_sums);
- if (sums16) {
- // the fp16 d of vector 8
- HVX_VectorPair vd01_sf = hvx_vec_f16_to_f32(vd01_hf);
- HVX_VectorPair vd23_sf = hvx_vec_f16_to_f32(vd23_hf);
- vd0_sf = Q6_V_lo_W(vd01_sf);
- vd1_sf = Q6_V_hi_W(vd01_sf);
- vd2_sf = Q6_V_lo_W(vd23_sf);
- vd3_sf = Q6_V_hi_W(vd23_sf);
- }
HVX_Vector voff0_sf = hvx_vec_mul_f32_f32(vd0_sf, v_sums_sf);
HVX_Vector voff1_sf = hvx_vec_mul_f32_f32(vd1_sf, Q6_V_vror_VR(v_sums_sf, 32));
HVX_Vector voff2_sf = hvx_vec_mul_f32_f32(vd2_sf, Q6_V_vror_VR(v_sums_sf, 64));
@@ -91,12 +94,12 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 64)),
};
if (sums16) {
- // the k 16..31 sums sit 4 halfwords after the k 0..15 sums
- const HVX_VectorPred q_lo = Q6_Q_vsetq_R(64);
- r_offset[0] = Q6_V_vmux_QVV(q_lo, r_offset[0], hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 8)));
- r_offset[1] = Q6_V_vmux_QVV(q_lo, r_offset[1], hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 72)));
- r_offset[2] = Q6_V_vmux_QVV(q_lo, r_offset[2], hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 8)));
- r_offset[3] = Q6_V_vmux_QVV(q_lo, r_offset[3], hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 72)));
+ // |sum of 16 int8| <= 2032 fits a halfword
+ int32_t __attribute__((aligned(128))) ws[32];
+ *(HVX_Vector *) ws = v_sums;
+ for (int b = 0; b < 4; b++) {
+ r_offset[b] = Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF));
+ }
}
static const uint8_t __attribute__((aligned(128))) repl[128] = {
@@ -138,11 +141,11 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
}
static inline void quantize_block_f32_q8_1_tiled(float * restrict x, uint8_t * restrict y_block) {
- quantize_block_f32_q8_1_tiled_impl(x, y_block, false);
+ quantize_block_f32_q8_1_tiled_impl(x, y_block, false, NULL);
}
-static inline void quantize_block_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y_block) {
- quantize_block_f32_q8_1_tiled_impl(x, y_block, true);
+static inline void quantize_block_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y_block, const float * restrict x_sb) {
+ quantize_block_f32_q8_1_tiled_impl(x, y_block, true, x_sb);
}
static inline void quantize_block_f32_q8_0_tiled(float * restrict x, uint8_t * restrict y_block) {
@@ -267,7 +270,7 @@ static void quantize_row_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restri
for (uint32_t i = 0; i < nb; i++) {
uint8_t * restrict y_block = y + i * 4 * 1280;
- quantize_block_f32_q8_1_s16_tiled(x + i * qk, y_block);
+ quantize_block_f32_q8_1_s16_tiled(x + i * qk, y_block, x + (i & ~1u) * qk); // k % 256 == 0 for Q2_K
}
}
@@ -655,7 +658,7 @@ static inline void accum_q3_k_32x2(
*v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo);
}
-// Q2_K (x = D * q + M): the Q3_K dot products without the -4 flags, M uses the q8_1_s16 sums
+// Q2_K: the Q3_K dot products without the -4 flags, scales and mins are applied in q2_k_int_accum
static inline HVX_VectorPair accum_q2_k_32x1(
const HVX_Vector * restrict vptr,
const HVX_Vector * restrict v_act
@@ -702,11 +705,57 @@ static inline void accum_q2_k_32x2(
*v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo);
}
-// D as the Q6_K scales, plus M (vector 3) times the per-16 activation sums in v_act[9]
-static inline HVX_Vector scale_q2_k_32x1(HVX_VectorPair v_sums, const HVX_Vector * restrict vptr, const HVX_Vector * restrict v_act) {
- HVX_VectorPair v_m = hvx_vec_mul_f16_f16_to_f32_pair(vptr[3], v_act[9]);
- HVX_Vector v_d = scale_q6_k_32x1(v_sums, vptr[2], v_act[8]);
- return hvx_vec_add_f32_f32(v_d, hvx_vec_add_f32_f32(Q6_V_lo_W(v_m), Q6_V_hi_W(v_m)));
+// Q2_K: fp16 d (*v_d) and -dmin (*v_m) of the 32 rows of a super-block (8 tiles, VTCM stride 384), each in both vector halves
+static inline void q2_k_load_dmin(const uint8_t * restrict sb, HVX_Vector * restrict v_d, HVX_Vector * restrict v_m) {
+ HVX_Vector v_dd = Q6_V_vzero(); // lanes 0..31: d, lanes 32..63: -dmin
+
+ #pragma unroll
+ for (int j = 0; j < 8; j++) {
+ HVX_Vector v = hvx_vmem(sb + j * 384 + 256); // the slice is at bytes 64..79
+ v_dd = Q6_V_vmux_QVV(Q6_Q_vsetq_R(16 * j), v_dd, Q6_V_vror_VR(v, (64 - 16 * j) & 127));
+ }
+
+ HVX_VectorPred q_lo = Q6_Q_vsetq_R(64);
+ HVX_Vector v_r = Q6_V_vror_VR(v_dd, 64);
+ *v_d = Q6_V_vmux_QVV(q_lo, v_dd, v_r);
+ *v_m = Q6_V_vmux_QVV(q_lo, v_r, v_dd);
+}
+
+// Q2_K: fp16 D = d * scale and M = -dmin * min of a tile from its scale bytes, in the lanes of the Q6_K scales
+static inline void q2_k_unpack_scales(const uint8_t * restrict tile, HVX_Vector v_d, HVX_Vector v_m,
+ HVX_Vector * restrict v_D, HVX_Vector * restrict v_M) {
+ HVX_Vector v_sc = hvx_vmem(tile + 256);
+ HVX_Vector v_l = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(Q6_V_vand_VV(v_sc, Q6_Vb_vsplat_R(0x0F))));
+ HVX_Vector v_mn = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(Q6_Vub_vlsr_VubR(v_sc, 4)));
+ // qf32 product: rounds like GGML_FP32_TO_FP16(d * scale) (qf16 is off by 1 ulp for many values on v73)
+ *v_D = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(v_l), v_d));
+ *v_M = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(v_mn), v_m));
+}
+
+// Q2_K scale and min nibbles of a tile as halfword pairs: word r = sub-block 1 << 16 | sub-block 0 of row r
+static inline void q2_k_scale_pairs(const uint8_t * restrict tile, HVX_Vector * restrict v_sc2, HVX_Vector * restrict v_mn2) {
+ HVX_Vector v_h = Q6_Vh_vshuff_Vh(Q6_V_lo_W(Q6_Wuh_vunpack_Vub(hvx_vmem(tile + 256))));
+ *v_sc2 = Q6_V_vand_VV(v_h, Q6_Vh_vsplat_R(0x000F));
+ *v_mn2 = Q6_Vuh_vlsr_VuhR(v_h, 4);
+}
+
+// Q2_K with q8_1_s16 activations, one tile: isum += scale * (q . a), msum += min * sum(a), per row, in int32.
+// |sum of 16 q * a| <= 3 * 128 * 16 fits a halfword, so both sub-blocks go through one vdmpy (no saturation)
+static inline void q2_k_int_accum(HVX_VectorPair v_sums, HVX_Vector v_sc2, HVX_Vector v_mn2, HVX_Vector v_bsum,
+ HVX_Vector * restrict v_isum, HVX_Vector * restrict v_msum) {
+ HVX_Vector v_s2 = Q6_Vh_vshuffe_VhVh(Q6_V_hi_W(v_sums), Q6_V_lo_W(v_sums));
+ *v_isum = Q6_Vw_vadd_VwVw(*v_isum, Q6_Vw_vdmpy_VhVh_sat(v_s2, v_sc2));
+ *v_msum = Q6_Vw_vadd_VwVw(*v_msum, Q6_Vw_vdmpy_VhVh_sat(v_mn2, v_bsum));
+}
+
+// Q2_K super-block result: d_a * (d * isum - dmin * msum), with v_d / v_m the fp16 d / -dmin of q2_k_load_dmin
+static inline HVX_Vector q2_k_int_scale(HVX_Vector v_isum, HVX_Vector v_msum, HVX_Vector v_d, HVX_Vector v_m, HVX_Vector v_act8) {
+ HVX_Vector v_d32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_d));
+ HVX_Vector v_m32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_m));
+ HVX_Vector v_da32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_act8));
+ HVX_Vector v_t = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(v_d32, Q6_Vsf_equals_Vw(v_isum)),
+ hvx_vec_mul_f32_f32(v_m32, Q6_Vsf_equals_Vw(v_msum)));
+ return hvx_vec_mul_f32_f32(v_t, v_da32);
}
static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
@@ -1174,12 +1223,22 @@ static void tiled_vec_dot_q2_k_32x1(const uint32_t n, float * restrict s, const
HVX_Vector v_sum_float = Q6_V_vzero();
uint32_t n_k_tiles = n / 32;
- for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
- const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 512);
- const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1280);
-
- HVX_VectorPair v_sums = accum_q2_k_32x1(vptr, v_act);
- v_sum_float = hvx_vec_add_f32_f32(v_sum_float, scale_q2_k_32x1(v_sums, vptr, v_act));
+ for (uint32_t kb = 0; kb < n_k_tiles; kb += 8) {
+ HVX_Vector v_d, v_m;
+ q2_k_load_dmin(tile_ptr + kb * 384, &v_d, &v_m);
+
+ HVX_Vector v_isum = Q6_V_vzero();
+ HVX_Vector v_msum = Q6_V_vzero();
+ for (uint32_t kt = kb; kt < kb + 8; kt++) {
+ const uint8_t * restrict tile = tile_ptr + kt * 384;
+ const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1280);
+
+ HVX_Vector v_sc2, v_mn2;
+ q2_k_scale_pairs(tile, &v_sc2, &v_mn2);
+ q2_k_int_accum(accum_q2_k_32x1((const HVX_Vector *) tile, v_act), v_sc2, v_mn2, v_act[9], &v_isum, &v_msum);
+ }
+ const HVX_Vector * restrict v_act_sb = (const HVX_Vector *) (y_q + kb * 1280);
+ v_sum_float = hvx_vec_add_f32_f32(v_sum_float, q2_k_int_scale(v_isum, v_msum, v_d, v_m, v_act_sb[8]));
}
if (sz) {
@@ -1198,16 +1257,31 @@ static void tiled_vec_dot_q2_k_32x2(const uint32_t n, float * restrict s0, float
HVX_Vector v_sum_float_c1 = Q6_V_vzero();
uint32_t n_k_tiles = n / 32;
- for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
- const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 512);
- const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1280);
- const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1280);
-
- HVX_VectorPair v_sums0, v_sums1;
- accum_q2_k_32x2(vptr, v_act0, v_act1, &v_sums0, &v_sums1);
-
- v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, scale_q2_k_32x1(v_sums0, vptr, v_act0));
- v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, scale_q2_k_32x1(v_sums1, vptr, v_act1));
+ for (uint32_t kb = 0; kb < n_k_tiles; kb += 8) {
+ HVX_Vector v_d, v_m;
+ q2_k_load_dmin(tile_ptr + kb * 384, &v_d, &v_m);
+
+ HVX_Vector v_isum0 = Q6_V_vzero();
+ HVX_Vector v_msum0 = Q6_V_vzero();
+ HVX_Vector v_isum1 = Q6_V_vzero();
+ HVX_Vector v_msum1 = Q6_V_vzero();
+ for (uint32_t kt = kb; kt < kb + 8; kt++) {
+ const uint8_t * restrict tile = tile_ptr + kt * 384;
+ const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1280);
+ const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1280);
+
+ HVX_Vector v_sc2, v_mn2;
+ q2_k_scale_pairs(tile, &v_sc2, &v_mn2);
+
+ HVX_VectorPair v_sums0, v_sums1;
+ accum_q2_k_32x2((const HVX_Vector *) tile, v_act0, v_act1, &v_sums0, &v_sums1);
+ q2_k_int_accum(v_sums0, v_sc2, v_mn2, v_act0[9], &v_isum0, &v_msum0);
+ q2_k_int_accum(v_sums1, v_sc2, v_mn2, v_act1[9], &v_isum1, &v_msum1);
+ }
+ const HVX_Vector * restrict v_act0_sb = (const HVX_Vector *) (y0_q + kb * 1280);
+ const HVX_Vector * restrict v_act1_sb = (const HVX_Vector *) (y1_q + kb * 1280);
+ v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, q2_k_int_scale(v_isum0, v_msum0, v_d, v_m, v_act0_sb[8]));
+ v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, q2_k_int_scale(v_isum1, v_msum1, v_d, v_m, v_act1_sb[8]));
}
if (sz0) {
@@ -1539,7 +1613,7 @@ static inline void quantize_f32_q8_1_s16_tiled_block_kernel(
const float * restrict src_ptr = (const float *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(float));
uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 1280;
- quantize_block_f32_q8_1_s16_tiled((float *) src_ptr, dst_ptr);
+ quantize_block_f32_q8_1_s16_tiled((float *) src_ptr, dst_ptr, src_ptr - (c & 1) * qk); // super-block of 256 k
c++;
if (c == nb) {
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c
index 1a513d160..28aed4680 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.c
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c
@@ -543,7 +543,7 @@ MATMUL_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_do
MATMUL_2D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x2, tiled_vec_dot_q6_k_32x1)
MATMUL_2D_REPACKED_IMPL(q5_k, 768, tiled_vec_dot_q5_k_32x2, tiled_vec_dot_q5_k_32x1)
MATMUL_2D_REPACKED_IMPL(q3_k, 512, tiled_vec_dot_q3_k_32x2, tiled_vec_dot_q3_k_32x1)
-MATMUL_2D_REPACKED_IMPL(q2_k, 512, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1)
+MATMUL_2D_REPACKED_IMPL(q2_k, 336, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1)
MATMUL_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1)
MATMUL_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1)
@@ -761,7 +761,7 @@ MATVEC_2D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x1)
MATVEC_2D_REPACKED_IMPL(q5_k, 768, tiled_vec_dot_q5_k_32x1)
MATVEC_2D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x1)
MATVEC_2D_REPACKED_IMPL(q3_k, 512, tiled_vec_dot_q3_k_32x1)
-MATVEC_2D_REPACKED_IMPL(q2_k, 512, tiled_vec_dot_q2_k_32x1)
+MATVEC_2D_REPACKED_IMPL(q2_k, 336, tiled_vec_dot_q2_k_32x1)
MATVEC_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x1)
MATVEC_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x1)
@@ -772,7 +772,7 @@ MATMUL_NX_2D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_do
MATMUL_NX_2D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1)
MATMUL_NX_2D_REPACKED_IMPL(q5_k, 768, tiled_vec_dot_q5_k_32x2, tiled_vec_dot_q5_k_32x1)
MATMUL_NX_2D_REPACKED_IMPL(q3_k, 512, tiled_vec_dot_q3_k_32x2, tiled_vec_dot_q3_k_32x1)
-MATMUL_NX_2D_REPACKED_IMPL(q2_k, 512, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1)
+MATMUL_NX_2D_REPACKED_IMPL(q2_k, 336, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1)
#define MATMUL_4D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1) \
static void hvx_mm_4d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
@@ -907,7 +907,7 @@ MATMUL_4D_REPACKED_IMPL(q8_0, 1088, tiled_vec_dot_q8_0_32x2, tiled_vec_do
MATMUL_4D_REPACKED_IMPL(q6_k, 896, tiled_vec_dot_q6_k_32x2, tiled_vec_dot_q6_k_32x1)
MATMUL_4D_REPACKED_IMPL(q5_k, 768, tiled_vec_dot_q5_k_32x2, tiled_vec_dot_q5_k_32x1)
MATMUL_4D_REPACKED_IMPL(q3_k, 512, tiled_vec_dot_q3_k_32x2, tiled_vec_dot_q3_k_32x1)
-MATMUL_4D_REPACKED_IMPL(q2_k, 512, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1)
+MATMUL_4D_REPACKED_IMPL(q2_k, 336, tiled_vec_dot_q2_k_32x2, tiled_vec_dot_q2_k_32x1)
MATMUL_4D_REPACKED_IMPL(iq4nl, 576, tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1)
MATMUL_4D_REPACKED_IMPL(mxfp4, 544, tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1)
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.h b/ggml/src/ggml-hexagon/htp/matmul-ops.h
index 1d11a4c12..522d8779f 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.h
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.h
@@ -38,11 +38,11 @@ extern "C" {
// vector 2: bit g set where the hmask bit of group g is clear (quant = low 2 bits - 4)
// vector 3: fp16 scales per row, d * (scales[] - 32): k 0..15 in lanes 0..31, k 16..31 in lanes 32..63
#define HTP_MM_WEIGHT_TILE_SIZE_Q3_K 512
-// Q2_K native 2-bit tile, vrmpy-ready like Q6_K
-// vectors 0..1: unsigned 2-bit quants, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6
-// vector 2: fp16 scales per row, d * (scales[] & 0xF), same lanes as Q3_K vector 3
-// vector 3: fp16 offsets per row, -dmin * (scales[] >> 4), same lanes
-#define HTP_MM_WEIGHT_TILE_SIZE_Q2_K 512
+// Q2_K native 2-bit tile with the ggml scale bytes, vrmpy-ready like Q6_K (same bits per weight as block_q2_K)
+// bytes 0..255: unsigned 2-bit quants, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6
+// bytes 256..319: scales[] bytes (min << 4 | scale), sub-block s of row r at 32 * s + r
+// bytes 320..335: 8 fp16, tile j of a super-block: d (j < 4) or -dmin (j >= 4) of rows 8 * (j % 4) .. 8 * (j % 4) + 7
+#define HTP_MM_WEIGHT_TILE_SIZE_Q2_K 336
// --- Weight Repacked Aligned Tile Sizes ---
#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0 640
@@ -53,7 +53,7 @@ extern "C" {
#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q5_K 768
#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K 896
#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q3_K 512
-#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_K 512
+#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_K 384
// --- Activation Tiled Block Sizes (including padding) ---
#define HTP_MM_ACT_TILE_SIZE_Q8_0 1152