Commit 76f186df7 for llama.cpp

commit 76f186df71a41abb3ee20fffedee31ca5dbc7d9f
Author: Bartowski <3266127+bartowski1182@users.noreply.github.com>
Date:   Thu Nov 19 03:04:58 2026 -0500

    ggml : IQ2 creation speedup through caching (#31910)

    * IQ2 quant creation speedup from caching

    * Rename variables for clarity

    * Explicit casts

    * Initialize all values, add comment to tolerance

diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c
index 3586f67ae..40ffe1e6b 100644
--- a/ggml/src/ggml-quants.c
+++ b/ggml/src/ggml-quants.c
@@ -3307,6 +3307,85 @@ static int iq2_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, co
     return grid_index;
 }

+// same result as iq2_find_best_neighbour: score = s*(s*quad_coef - lin_coef) + const, and the coefs do not depend on the scale s, so cache them per pattern
+// take the cached winner only if all other neighbours are worse by more than the float rounding bound of the exact search, else run the exact search
+#define IQ2_NEIGHBOUR_CACHE_MAX 512
+
+typedef struct {
+    int pattern;   // pattern whose coefs are cached, -1 if none
+    int last_best; // last winner for this pattern
+    // sums for the rounding bound: |w|, |w|*|x|, |w|*x^2, and the same with 1 + |w|
+    double sum_w, sum_wx, sum_wxx, sum_w1, sum_w1x, sum_w1xx;
+    double lin_term[8][8], quad_term[8][8]; // 2*w*x*q and w*q*q per element, indexed by the grid level q
+    double lin_coef[IQ2_NEIGHBOUR_CACHE_MAX];
+    double quad_coef[IQ2_NEIGHBOUR_CACHE_MAX];
+} iq2_neighbour_cache;
+
+static void iq2_neighbour_cache_init(iq2_neighbour_cache * cache, const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight) {
+    cache->pattern = -1;
+    cache->sum_w = cache->sum_wx = cache->sum_wxx = cache->sum_w1 = cache->sum_w1x = cache->sum_w1xx = 0;
+    for (int i = 0; i < 8; ++i) {
+        const double w = (double)weight[i], x = (double)xval[i], abs_w = fabs(w), abs_x = fabs(x);
+        cache->sum_w  += abs_w;     cache->sum_wx  += abs_w*abs_x;       cache->sum_wxx  += abs_w*abs_x*abs_x;
+        cache->sum_w1 += 1 + abs_w; cache->sum_w1x += (1 + abs_w)*abs_x; cache->sum_w1xx += (1 + abs_w)*abs_x*abs_x;
+        for (int q = 0; q < 8; ++q) {
+            cache->lin_term[i][q]  = 2*w*x*q;
+            cache->quad_term[i][q] = w*q*q;
+        }
+    }
+}
+
+static int iq2_count_at_most(const double * GGML_RESTRICT values, int n, double limit) {
+    int count = 0;
+    for (int j = 0; j < n; ++j) count += values[j] <= limit;
+    return count;
+}
+
+static int iq2_find_best_neighbour_cached(iq2_neighbour_cache * cache, int pattern, const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid,
+        const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, int8_t * GGML_RESTRICT L) {
+    const int num_neighbors = neighbours[0];
+    const double scale_d = (double)scale;
+    const double max_level = 7*scale_d;
+    // size of the numbers in the exact search, its rounding error is a multiple of 2^-24*size (+ size_subnormal for underflow)
+    const double size           = max_level*max_level*cache->sum_w + 2*max_level*cache->sum_wx + cache->sum_wxx;
+    const double size_subnormal = (1 + max_level)*(1 + max_level)*cache->sum_w1 + 2*(1 + max_level)*cache->sum_w1x + cache->sum_w1xx;
+    // the cached score is (exact score - const)/s, so the 2^-18*size margin on the exact score is divided by s
+    const double tolerance = (0x1p-18*size + 0x1p-100*size_subnormal)/scale_d;
+    if (num_neighbors <= 0 || num_neighbors > IQ2_NEIGHBOUR_CACHE_MAX || !(scale_d > 0 && size < 0x1p100 && size_subnormal < 0x1p100 && tolerance > 0)) {
+        return iq2_find_best_neighbour(neighbours, grid, xval, weight, scale, L);
+    }
+    if (cache->pattern != pattern) {
+        for (int j = 0; j < num_neighbors; ++j) {
+            const int8_t * levels = (const int8_t *)(grid + neighbours[j+1]);
+            double lin = 0, quad = 0;
+            for (int i = 0; i < 8; ++i) {
+                lin  += cache->lin_term[i][levels[i]];
+                quad += cache->quad_term[i][levels[i]];
+            }
+            cache->lin_coef[j] = lin; cache->quad_coef[j] = quad;
+        }
+        cache->pattern = pattern;
+        cache->last_best = 0;
+    }
+    double score[IQ2_NEIGHBOUR_CACHE_MAX];
+    for (int j = 0; j < num_neighbors; ++j) score[j] = scale_d*cache->quad_coef[j] - cache->lin_coef[j];
+    // the winner must be the only neighbour within tolerance, try the previous winner for this pattern first
+    int best = cache->last_best;
+    if (iq2_count_at_most(score, num_neighbors, score[best] + tolerance) != 1) {
+        for (int j = 0; j < num_neighbors; ++j) {
+            if (score[j] < score[best]) best = j;
+        }
+        if (iq2_count_at_most(score, num_neighbors, score[best] + tolerance) != 1) {
+            return iq2_find_best_neighbour(neighbours, grid, xval, weight, scale, L);
+        }
+        cache->last_best = best;
+    }
+    const int grid_index = neighbours[best+1];
+    const int8_t * levels = (const int8_t *)(grid + grid_index);
+    for (int i = 0; i < 8; ++i) L[i] = (levels[i] - 1)/2;
+    return grid_index;
+}
+
 static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights) {

     const int gindex = iq2_data_index(GGML_TYPE_IQ2_XXS);
@@ -3335,6 +3414,8 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
     float  waux[32];
     uint8_t block_signs[4];
     uint32_t q2[2*(QK_K/32)];
+    iq2_neighbour_cache neighbour_cache[4];
+    int8_t Laux_prev[32];

     for (int ibl = 0; ibl < nbl; ++ibl) {

@@ -3390,6 +3471,7 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
                 continue;
             }
             float best = 0;
+            for (int k = 0; k < 4; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
             for (int is = -6; is <= 6; ++is) {
                 float id = (2*kMaxQ-1+is*0.1f)/eff_max;
                 float this_scale = 1/id;
@@ -3403,9 +3485,14 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
                     int grid_index = kmap_q2xs[u];
                     if (grid_index < 0) {
                         const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
-                        grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
+                        grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
                     }
                 }
+                // same candidate as for the previous scale: it cannot change the result
+                if (is > -6 && memcmp(Laux, Laux_prev, 32) == 0) {
+                    continue;
+                }
+                memcpy(Laux_prev, Laux, 32);
                 float sumqx = 0, sumq2 = 0;
                 for (int i = 0; i < 32; ++i) {
                     float w = weight[i];
@@ -3430,7 +3517,7 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
                     int grid_index = kmap_q2xs[u];
                     if (grid_index < 0) {
                         const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
-                        grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
+                        grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
                     }
                     const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index);
                     for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2;
@@ -3515,6 +3602,9 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
     bool   is_on_grid_aux[2];
     uint8_t block_signs[2];
     uint16_t q2[2*(QK_K/16)];
+    iq2_neighbour_cache neighbour_cache[2];
+    int8_t Laux_prev[16];
+    bool   is_on_grid_prev[2];

     for (int ibl = 0; ibl < nbl; ++ibl) {

@@ -3566,6 +3656,7 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
             float best = 0;
             float scale = max/(2*kMaxQ-1);
             is_on_grid[0] = is_on_grid[1] = true;
+            for (int k = 0; k < 2; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
             for (int is = -9; is <= 9; ++is) {
                 float id = (2*kMaxQ-1+is*0.1f)/max;
                 float this_scale = 1/id;
@@ -3581,9 +3672,15 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
                     if (grid_index < 0) {
                         is_on_grid_aux[k] = false;
                         const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
-                        grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
+                        grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
                     }
                 }
+                // same candidate as for the previous scale: it cannot change the result
+                if (is > -9 && memcmp(Laux, Laux_prev, 16) == 0 && is_on_grid_aux[0] == is_on_grid_prev[0] && is_on_grid_aux[1] == is_on_grid_prev[1]) {
+                    continue;
+                }
+                memcpy(Laux_prev, Laux, 16);
+                is_on_grid_prev[0] = is_on_grid_aux[0]; is_on_grid_prev[1] = is_on_grid_aux[1];
                 float sumqx = 0, sumq2 = 0;
                 for (int i = 0; i < 16; ++i) {
                     float w = weight[i];
@@ -3613,7 +3710,7 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
                     int grid_index = kmap_q2xs[u];
                     if (grid_index < 0) {
                         const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
-                        grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
+                        grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
                     }
                 }
                 float sumqx = 0, sumq2 = 0;
@@ -5155,6 +5252,9 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
     bool   is_on_grid[2];
     bool   is_on_grid_aux[2];
     uint8_t block_signs[2];
+    iq2_neighbour_cache neighbour_cache[2];
+    int8_t Laux_prev[16];
+    bool   is_on_grid_prev[2];

     for (int ibl = 0; ibl < nbl; ++ibl) {

@@ -5197,6 +5297,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
             float best = 0;
             float scale = max/(2*kMaxQ-1);
             is_on_grid[0] = is_on_grid[1] = true;
+            for (int k = 0; k < 2; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
             for (int is = -9; is <= 9; ++is) {
                 float id = (2*kMaxQ-1+is*0.1f)/max;
                 float this_scale = 1/id;
@@ -5212,9 +5313,15 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
                     if (grid_index < 0) {
                         is_on_grid_aux[k] = false;
                         const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
-                        grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
+                        grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
                     }
                 }
+                // same candidate as for the previous scale: it cannot change the result
+                if (is > -9 && memcmp(Laux, Laux_prev, 16) == 0 && is_on_grid_aux[0] == is_on_grid_prev[0] && is_on_grid_aux[1] == is_on_grid_prev[1]) {
+                    continue;
+                }
+                memcpy(Laux_prev, Laux, 16);
+                is_on_grid_prev[0] = is_on_grid_aux[0]; is_on_grid_prev[1] = is_on_grid_aux[1];
                 float sumqx = 0, sumq2 = 0;
                 for (int i = 0; i < 16; ++i) {
                     float w = weight[i];
@@ -5244,7 +5351,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
                     int grid_index = kmap_q2xs[u];
                     if (grid_index < 0) {
                         const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
-                        grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
+                        grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
                     }
                 }
                 float sumqx = 0, sumq2 = 0;