Commit 76f186df7 for llama.cpp
commit 76f186df71a41abb3ee20fffedee31ca5dbc7d9f
Author: Bartowski <3266127+bartowski1182@users.noreply.github.com>
Date: Thu Nov 19 03:04:58 2026 -0500
ggml : IQ2 creation speedup through caching (#31910)
* IQ2 quant creation speedup from caching
* Rename variables for clarity
* Explicit casts
* Initialize all values, add comment to tolerance
diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c
index 3586f67ae..40ffe1e6b 100644
--- a/ggml/src/ggml-quants.c
+++ b/ggml/src/ggml-quants.c
@@ -3307,6 +3307,85 @@ static int iq2_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, co
return grid_index;
}
+// same result as iq2_find_best_neighbour: score = s*(s*quad_coef - lin_coef) + const, and the coefs do not depend on the scale s, so cache them per pattern
+// take the cached winner only if all other neighbours are worse by more than the float rounding bound of the exact search, else run the exact search
+#define IQ2_NEIGHBOUR_CACHE_MAX 512
+
+typedef struct {
+ int pattern; // pattern whose coefs are cached, -1 if none
+ int last_best; // last winner for this pattern
+ // sums for the rounding bound: |w|, |w|*|x|, |w|*x^2, and the same with 1 + |w|
+ double sum_w, sum_wx, sum_wxx, sum_w1, sum_w1x, sum_w1xx;
+ double lin_term[8][8], quad_term[8][8]; // 2*w*x*q and w*q*q per element, indexed by the grid level q
+ double lin_coef[IQ2_NEIGHBOUR_CACHE_MAX];
+ double quad_coef[IQ2_NEIGHBOUR_CACHE_MAX];
+} iq2_neighbour_cache;
+
+static void iq2_neighbour_cache_init(iq2_neighbour_cache * cache, const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight) {
+ cache->pattern = -1;
+ cache->sum_w = cache->sum_wx = cache->sum_wxx = cache->sum_w1 = cache->sum_w1x = cache->sum_w1xx = 0;
+ for (int i = 0; i < 8; ++i) {
+ const double w = (double)weight[i], x = (double)xval[i], abs_w = fabs(w), abs_x = fabs(x);
+ cache->sum_w += abs_w; cache->sum_wx += abs_w*abs_x; cache->sum_wxx += abs_w*abs_x*abs_x;
+ cache->sum_w1 += 1 + abs_w; cache->sum_w1x += (1 + abs_w)*abs_x; cache->sum_w1xx += (1 + abs_w)*abs_x*abs_x;
+ for (int q = 0; q < 8; ++q) {
+ cache->lin_term[i][q] = 2*w*x*q;
+ cache->quad_term[i][q] = w*q*q;
+ }
+ }
+}
+
+static int iq2_count_at_most(const double * GGML_RESTRICT values, int n, double limit) {
+ int count = 0;
+ for (int j = 0; j < n; ++j) count += values[j] <= limit;
+ return count;
+}
+
+static int iq2_find_best_neighbour_cached(iq2_neighbour_cache * cache, int pattern, const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid,
+ const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, int8_t * GGML_RESTRICT L) {
+ const int num_neighbors = neighbours[0];
+ const double scale_d = (double)scale;
+ const double max_level = 7*scale_d;
+ // size of the numbers in the exact search, its rounding error is a multiple of 2^-24*size (+ size_subnormal for underflow)
+ const double size = max_level*max_level*cache->sum_w + 2*max_level*cache->sum_wx + cache->sum_wxx;
+ const double size_subnormal = (1 + max_level)*(1 + max_level)*cache->sum_w1 + 2*(1 + max_level)*cache->sum_w1x + cache->sum_w1xx;
+ // the cached score is (exact score - const)/s, so the 2^-18*size margin on the exact score is divided by s
+ const double tolerance = (0x1p-18*size + 0x1p-100*size_subnormal)/scale_d;
+ if (num_neighbors <= 0 || num_neighbors > IQ2_NEIGHBOUR_CACHE_MAX || !(scale_d > 0 && size < 0x1p100 && size_subnormal < 0x1p100 && tolerance > 0)) {
+ return iq2_find_best_neighbour(neighbours, grid, xval, weight, scale, L);
+ }
+ if (cache->pattern != pattern) {
+ for (int j = 0; j < num_neighbors; ++j) {
+ const int8_t * levels = (const int8_t *)(grid + neighbours[j+1]);
+ double lin = 0, quad = 0;
+ for (int i = 0; i < 8; ++i) {
+ lin += cache->lin_term[i][levels[i]];
+ quad += cache->quad_term[i][levels[i]];
+ }
+ cache->lin_coef[j] = lin; cache->quad_coef[j] = quad;
+ }
+ cache->pattern = pattern;
+ cache->last_best = 0;
+ }
+ double score[IQ2_NEIGHBOUR_CACHE_MAX];
+ for (int j = 0; j < num_neighbors; ++j) score[j] = scale_d*cache->quad_coef[j] - cache->lin_coef[j];
+ // the winner must be the only neighbour within tolerance, try the previous winner for this pattern first
+ int best = cache->last_best;
+ if (iq2_count_at_most(score, num_neighbors, score[best] + tolerance) != 1) {
+ for (int j = 0; j < num_neighbors; ++j) {
+ if (score[j] < score[best]) best = j;
+ }
+ if (iq2_count_at_most(score, num_neighbors, score[best] + tolerance) != 1) {
+ return iq2_find_best_neighbour(neighbours, grid, xval, weight, scale, L);
+ }
+ cache->last_best = best;
+ }
+ const int grid_index = neighbours[best+1];
+ const int8_t * levels = (const int8_t *)(grid + grid_index);
+ for (int i = 0; i < 8; ++i) L[i] = (levels[i] - 1)/2;
+ return grid_index;
+}
+
static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights) {
const int gindex = iq2_data_index(GGML_TYPE_IQ2_XXS);
@@ -3335,6 +3414,8 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
float waux[32];
uint8_t block_signs[4];
uint32_t q2[2*(QK_K/32)];
+ iq2_neighbour_cache neighbour_cache[4];
+ int8_t Laux_prev[32];
for (int ibl = 0; ibl < nbl; ++ibl) {
@@ -3390,6 +3471,7 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
continue;
}
float best = 0;
+ for (int k = 0; k < 4; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
for (int is = -6; is <= 6; ++is) {
float id = (2*kMaxQ-1+is*0.1f)/eff_max;
float this_scale = 1/id;
@@ -3403,9 +3485,14 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
int grid_index = kmap_q2xs[u];
if (grid_index < 0) {
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
- grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
+ grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
}
}
+ // same candidate as for the previous scale: it cannot change the result
+ if (is > -6 && memcmp(Laux, Laux_prev, 32) == 0) {
+ continue;
+ }
+ memcpy(Laux_prev, Laux, 32);
float sumqx = 0, sumq2 = 0;
for (int i = 0; i < 32; ++i) {
float w = weight[i];
@@ -3430,7 +3517,7 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
int grid_index = kmap_q2xs[u];
if (grid_index < 0) {
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
- grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
+ grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
}
const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index);
for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2;
@@ -3515,6 +3602,9 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
bool is_on_grid_aux[2];
uint8_t block_signs[2];
uint16_t q2[2*(QK_K/16)];
+ iq2_neighbour_cache neighbour_cache[2];
+ int8_t Laux_prev[16];
+ bool is_on_grid_prev[2];
for (int ibl = 0; ibl < nbl; ++ibl) {
@@ -3566,6 +3656,7 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
float best = 0;
float scale = max/(2*kMaxQ-1);
is_on_grid[0] = is_on_grid[1] = true;
+ for (int k = 0; k < 2; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
for (int is = -9; is <= 9; ++is) {
float id = (2*kMaxQ-1+is*0.1f)/max;
float this_scale = 1/id;
@@ -3581,9 +3672,15 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
if (grid_index < 0) {
is_on_grid_aux[k] = false;
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
- grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
+ grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
}
}
+ // same candidate as for the previous scale: it cannot change the result
+ if (is > -9 && memcmp(Laux, Laux_prev, 16) == 0 && is_on_grid_aux[0] == is_on_grid_prev[0] && is_on_grid_aux[1] == is_on_grid_prev[1]) {
+ continue;
+ }
+ memcpy(Laux_prev, Laux, 16);
+ is_on_grid_prev[0] = is_on_grid_aux[0]; is_on_grid_prev[1] = is_on_grid_aux[1];
float sumqx = 0, sumq2 = 0;
for (int i = 0; i < 16; ++i) {
float w = weight[i];
@@ -3613,7 +3710,7 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
int grid_index = kmap_q2xs[u];
if (grid_index < 0) {
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
- grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
+ grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
}
}
float sumqx = 0, sumq2 = 0;
@@ -5155,6 +5252,9 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
bool is_on_grid[2];
bool is_on_grid_aux[2];
uint8_t block_signs[2];
+ iq2_neighbour_cache neighbour_cache[2];
+ int8_t Laux_prev[16];
+ bool is_on_grid_prev[2];
for (int ibl = 0; ibl < nbl; ++ibl) {
@@ -5197,6 +5297,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
float best = 0;
float scale = max/(2*kMaxQ-1);
is_on_grid[0] = is_on_grid[1] = true;
+ for (int k = 0; k < 2; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
for (int is = -9; is <= 9; ++is) {
float id = (2*kMaxQ-1+is*0.1f)/max;
float this_scale = 1/id;
@@ -5212,9 +5313,15 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
if (grid_index < 0) {
is_on_grid_aux[k] = false;
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
- grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
+ grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
}
}
+ // same candidate as for the previous scale: it cannot change the result
+ if (is > -9 && memcmp(Laux, Laux_prev, 16) == 0 && is_on_grid_aux[0] == is_on_grid_prev[0] && is_on_grid_aux[1] == is_on_grid_prev[1]) {
+ continue;
+ }
+ memcpy(Laux_prev, Laux, 16);
+ is_on_grid_prev[0] = is_on_grid_aux[0]; is_on_grid_prev[1] = is_on_grid_aux[1];
float sumqx = 0, sumq2 = 0;
for (int i = 0; i < 16; ++i) {
float w = weight[i];
@@ -5244,7 +5351,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
int grid_index = kmap_q2xs[u];
if (grid_index < 0) {
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
- grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
+ grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
}
}
float sumqx = 0, sumq2 = 0;