Commit 04fc2a69e9 for aom

commit 04fc2a69e977441fc09d7fed0123197813e547bd
Author: Nithya V S <Nithya.VS@amd.com>
Date:   Fri Nov 13 17:45:48 2026 +0530

    Specialized simd coeff cost token fills

    Add specialized helpers for filling coeff cost tables from CDFs (for cdf
    2,4 batch). Also a LUT for low p15 values for cost_symbol. Enabled only
    for RT.

    Correctness and speed (disabled) tests are added.

    Kernel speed up (Zen 5c, AOCC 5.2.0), AVX2 vs C:
    cdf_4_batch base_cost (n=42, stride 8) : 3.22x
    cdf_4_batch br_rate   (n=21, stride 4) : 2.23x
    cdf_2_batch txb_skip  (n=13, stride 2) : 1.77x
    cdf_2_batch eob_extra (n= 9, stride 2) : 2.16x

    aomenc --rt q-mode, single thread, core-pinned, over AOM CTC:
    A3 - Johny, KristenAndSara, Vidyo3, Vidyo4; B1 (12 clips); B2 (11
    clips), 6 CQ (23-63)

      cpu-used :   5     6     7     8     9    10
      mean %   : 3.66  4.38  4.95  5.36  6.67  7.34    (overall 5.39)

    Change-Id: Ief2aa8ba8e58a7bc93100d40fdde16c3a8ba3a51

diff --git a/av1/av1.cmake b/av1/av1.cmake
index 654a147871..2f56db9250 100644
--- a/av1/av1.cmake
+++ b/av1/av1.cmake
@@ -372,6 +372,7 @@ list(APPEND AOM_AV1_ENCODER_INTRIN_SSE4_1

 list(APPEND AOM_AV1_ENCODER_INTRIN_AVX2
             "${AOM_ROOT}/av1/encoder/x86/av1_quantize_avx2.c"
+            "${AOM_ROOT}/av1/encoder/x86/cost_avx2.c"
             "${AOM_ROOT}/av1/encoder/x86/error_intrin_avx2.c"
             "${AOM_ROOT}/av1/encoder/x86/av1_fwd_txfm_avx2.h"
             "${AOM_ROOT}/av1/encoder/x86/av1_fwd_txfm2d_avx2.c"
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 41812ca69f..2e0dda53e9 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -307,6 +307,12 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
   add_proto qw/int64_t av1_block_error_lp/, "const int16_t *coeff, const int16_t *dqcoeff, intptr_t block_size";
   specialize qw/av1_block_error_lp sse2 avx2 neon sve/;

+  add_proto qw/void av1_cost_tokens_from_cdf_2_batch/, "int *costs, int costs_stride, const uint16_t *cdf, int cdf_stride, int n";
+  specialize qw/av1_cost_tokens_from_cdf_2_batch avx2/;
+
+  add_proto qw/void av1_cost_tokens_from_cdf_4_batch/, "int *costs, int costs_stride, const uint16_t *cdf, int cdf_stride, int n";
+  specialize qw/av1_cost_tokens_from_cdf_4_batch avx2/;
+
   add_proto qw/void av1_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
   specialize qw/av1_quantize_fp sse2 avx2 neon/;

diff --git a/av1/encoder/cost.c b/av1/encoder/cost.c
index 6552c7ea9e..81988cfff8 100644
--- a/av1/encoder/cost.c
+++ b/av1/encoder/cost.c
@@ -10,7 +10,10 @@
  */
 #include <assert.h>

+#include "config/av1_rtcd.h"
+
 #include "aom_dsp/entcode.h"
+#include "aom_ports/aom_once.h"
 #include "av1/encoder/cost.h"
 #include "av1/common/entropy.h"

@@ -28,6 +31,15 @@ const uint16_t av1_prob_cost[129] = {
   23,  20,  18,  15,  12,  9,   6,   3,   3,
 };

+uint16_t av1_cost_lut_low[COST_LUT_LOW];
+
+static void init_cost_luts(void) {
+  for (int p = 1; p < COST_LUT_LOW; ++p)
+    av1_cost_lut_low[p] = (uint16_t)av1_cost_symbol((aom_cdf_prob)p);
+}
+
+void av1_init_cost_luts(void) { aom_once(init_cost_luts); }
+
 void av1_cost_tokens_from_cdf(int *costs, const aom_cdf_prob *cdf,
                               const int *inv_map) {
   int prev_icdf = CDF_PROB_TOP;
@@ -51,3 +63,21 @@ void av1_cost_tokens_from_cdf(int *costs, const aom_cdf_prob *cdf,
     }
   }
 }
+
+// Fills N=2 costs for `n` CDFs: costs[k*costs_stride + 0..1] gets the two
+// symbol costs of cdf[k*cdf_stride + 0..1]. inv_map is implicitly NULL.
+void av1_cost_tokens_from_cdf_2_batch_c(int *costs, int costs_stride,
+                                        const uint16_t *cdf, int cdf_stride,
+                                        int n) {
+  for (int i = 0; i < n; ++i)
+    av1_cost_tokens_from_cdf_2(costs + i * costs_stride, cdf + i * cdf_stride);
+}
+
+// Fills N=4 costs for `n` CDFs: costs[k*costs_stride + 0..3] gets the four
+// symbol costs of cdf[k*cdf_stride + 0..3]. inv_map is implicitly NULL.
+void av1_cost_tokens_from_cdf_4_batch_c(int *costs, int costs_stride,
+                                        const uint16_t *cdf, int cdf_stride,
+                                        int n) {
+  for (int i = 0; i < n; ++i)
+    av1_cost_tokens_from_cdf_4(costs + i * costs_stride, cdf + i * cdf_stride);
+}
diff --git a/av1/encoder/cost.h b/av1/encoder/cost.h
index cdadb893cc..5b108050b2 100644
--- a/av1/encoder/cost.h
+++ b/av1/encoder/cost.h
@@ -12,6 +12,7 @@
 #ifndef AOM_AV1_ENCODER_COST_H_
 #define AOM_AV1_ENCODER_COST_H_

+#include "aom_dsp/entcode.h"
 #include "aom_dsp/prob.h"
 #include "aom/aom_integer.h"

@@ -19,7 +20,12 @@
 extern "C" {
 #endif

+// Direct lookup table for small p15 (< COST_LUT_LOW).
+// COST_LUT_LOW * 2 B = 8 KB.
+#define COST_LUT_LOW 4096
+
 extern const uint16_t av1_prob_cost[129];
+extern uint16_t av1_cost_lut_low[COST_LUT_LOW];  // filled by av1_init_cost_luts

 // The factor to scale from cost in bits to cost in av1_prob_cost units.
 #define AV1_PROB_COST_SHIFT 9
@@ -47,8 +53,43 @@ static inline int av1_cost_symbol(aom_cdf_prob p15) {
   return av1_cost_symbol_clamped(clamped);
 }

+// Same as av1_cost_symbol(), but uses a lookup table for small p15.
+static inline int av1_cost_symbol_lut(aom_cdf_prob p15) {
+  if (p15 < COST_LUT_LOW) return av1_cost_lut_low[p15];
+  return av1_cost_symbol(p15);
+}
+
+// Same as av1_cost_tokens_from_cdf() for a 2-symbol CDF with inv_map == NULL.
+// The fixed symbol count removes the CDF-terminator check.
+static inline void av1_cost_tokens_from_cdf_2(int *costs,
+                                              const aom_cdf_prob *cdf) {
+  aom_cdf_prob prev = 0;
+  for (int i = 0; i < 2; ++i) {
+    const aom_cdf_prob c = AOM_ICDF(cdf[i]);
+    aom_cdf_prob p15 = c - prev;
+    p15 = (p15 < EC_MIN_PROB) ? EC_MIN_PROB : p15;
+    prev = c;
+    costs[i] = av1_cost_symbol_lut(p15);
+  }
+}
+
+// Same as av1_cost_tokens_from_cdf() for a 4-symbol CDF with inv_map == NULL.
+// The fixed symbol count removes the CDF-terminator check.
+static inline void av1_cost_tokens_from_cdf_4(int *costs,
+                                              const aom_cdf_prob *cdf) {
+  aom_cdf_prob prev = 0;
+  for (int i = 0; i < 4; ++i) {
+    const aom_cdf_prob c = AOM_ICDF(cdf[i]);
+    aom_cdf_prob p15 = c - prev;
+    p15 = (p15 < EC_MIN_PROB) ? EC_MIN_PROB : p15;
+    prev = c;
+    costs[i] = av1_cost_symbol_lut(p15);
+  }
+}
+
 void av1_cost_tokens_from_cdf(int *costs, const aom_cdf_prob *cdf,
                               const int *inv_map);
+void av1_init_cost_luts(void);  // one-time fill of av1_cost_lut_low

 #ifdef __cplusplus
 }  // extern "C"
diff --git a/av1/encoder/encodeframe_utils.c b/av1/encoder/encodeframe_utils.c
index e65f687d23..672815ba61 100644
--- a/av1/encoder/encodeframe_utils.c
+++ b/av1/encoder/encodeframe_utils.c
@@ -1688,7 +1688,8 @@ void av1_set_cost_upd_freq(AV1_COMP *cpi, ThreadData *td,
       if (skip_cost_update(cm->seq_params, tile_info, mi_row, mi_col,
                            cpi->sf.inter_sf.coeff_cost_upd_level))
         break;
-      av1_fill_coeff_costs(&x->coeff_costs, xd->tile_ctx, num_planes);
+      av1_fill_coeff_costs(&x->coeff_costs, xd->tile_ctx, num_planes,
+                           cpi->oxcf.mode == REALTIME);
       break;
     default: assert(0);
   }
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 6f8b9f5901..15aed5415c 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -302,6 +302,7 @@ void av1_initialize_enc(unsigned int usage, enum aom_rc_mode end_usage) {
   aom_scale_rtcd();
   av1_init_intra_predictors();
   av1_init_me_luts();
+  av1_init_cost_luts();
   if (!is_allintra) av1_init_wedge_masks();
   if (!is_allintra || end_usage != AOM_Q) av1_rc_init_minq_luts();
 }
diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 9abd00ccdd..23f5e77764 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -36,6 +36,7 @@
 #include "av1/encoder/ratectrl.h"
 #include "av1/encoder/rd.h"
 #include "config/aom_config.h"
+#include "config/av1_rtcd.h"

 #define RD_THRESH_POW 1.25

@@ -599,8 +600,10 @@ static void set_block_thresholds(const AV1_COMMON *cm, RD_OPT *rd,
   }
 }

+// In real-time mode, fill the coeff cost tables with the batched/SIMD
+// specializations; otherwise use the generic scalar path.
 void av1_fill_coeff_costs(CoeffCosts *coeff_costs, FRAME_CONTEXT *fc,
-                          const int num_planes) {
+                          const int num_planes, int is_realtime) {
   const int nplanes = AOMMIN(num_planes, PLANE_TYPES);
   for (int eob_multi_size = 0; eob_multi_size < 7; ++eob_multi_size) {
     for (int plane = 0; plane < nplanes; ++plane) {
@@ -626,17 +629,31 @@ void av1_fill_coeff_costs(CoeffCosts *coeff_costs, FRAME_CONTEXT *fc,
     for (int plane = 0; plane < nplanes; ++plane) {
       LV_MAP_COEFF_COST *pcost = &coeff_costs->coeff_costs[tx_size][plane];

-      for (int ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx)
-        av1_cost_tokens_from_cdf(pcost->txb_skip_cost[ctx],
-                                 fc->txb_skip_cdf[tx_size][ctx], NULL);
+      if (is_realtime) {
+        av1_cost_tokens_from_cdf_2_batch(pcost->txb_skip_cost[0], 2,
+                                         fc->txb_skip_cdf[tx_size][0],
+                                         CDF_SIZE(2), TXB_SKIP_CONTEXTS);
+      } else {
+        for (int ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx)
+          av1_cost_tokens_from_cdf(pcost->txb_skip_cost[ctx],
+                                   fc->txb_skip_cdf[tx_size][ctx], NULL);
+      }

       for (int ctx = 0; ctx < SIG_COEF_CONTEXTS_EOB; ++ctx)
         av1_cost_tokens_from_cdf(pcost->base_eob_cost[ctx],
                                  fc->coeff_base_eob_cdf[tx_size][plane][ctx],
                                  NULL);
-      for (int ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx)
-        av1_cost_tokens_from_cdf(pcost->base_cost[ctx],
-                                 fc->coeff_base_cdf[tx_size][plane][ctx], NULL);
+      // base_cost[ctx] is int[8]; only [0..3] are filled here (stride 8).
+      if (is_realtime) {
+        av1_cost_tokens_from_cdf_4_batch(pcost->base_cost[0], 8,
+                                         fc->coeff_base_cdf[tx_size][plane][0],
+                                         CDF_SIZE(4), SIG_COEF_CONTEXTS);
+      } else {
+        for (int ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx)
+          av1_cost_tokens_from_cdf(pcost->base_cost[ctx],
+                                   fc->coeff_base_cdf[tx_size][plane][ctx],
+                                   NULL);
+      }

       for (int ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx) {
         pcost->base_cost[ctx][4] = 0;
@@ -649,21 +666,37 @@ void av1_fill_coeff_costs(CoeffCosts *coeff_costs, FRAME_CONTEXT *fc,
             pcost->base_cost[ctx][3] - pcost->base_cost[ctx][2];
       }

-      for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx)
-        av1_cost_tokens_from_cdf(pcost->eob_extra_cost[ctx],
-                                 fc->eob_extra_cdf[tx_size][plane][ctx], NULL);
+      if (is_realtime) {
+        av1_cost_tokens_from_cdf_2_batch(pcost->eob_extra_cost[0], 2,
+                                         fc->eob_extra_cdf[tx_size][plane][0],
+                                         CDF_SIZE(2), EOB_COEF_CONTEXTS);
+      } else {
+        for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx)
+          av1_cost_tokens_from_cdf(pcost->eob_extra_cost[ctx],
+                                   fc->eob_extra_cdf[tx_size][plane][ctx],
+                                   NULL);
+      }

       for (int ctx = 0; ctx < DC_SIGN_CONTEXTS; ++ctx)
         av1_cost_tokens_from_cdf(pcost->dc_sign_cost[ctx],
                                  fc->dc_sign_cdf[plane][ctx], NULL);

+      int br_rate_all[LEVEL_CONTEXTS][BR_CDF_SIZE];
+      if (is_realtime) {
+        av1_cost_tokens_from_cdf_4_batch(
+            br_rate_all[0], BR_CDF_SIZE,
+            fc->coeff_br_cdf[AOMMIN(tx_size, TX_32X32)][plane][0],
+            CDF_SIZE(BR_CDF_SIZE), LEVEL_CONTEXTS);
+      } else {
+        for (int ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx)
+          av1_cost_tokens_from_cdf(
+              br_rate_all[ctx],
+              fc->coeff_br_cdf[AOMMIN(tx_size, TX_32X32)][plane][ctx], NULL);
+      }
       for (int ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx) {
-        int br_rate[BR_CDF_SIZE];
+        const int *br_rate = br_rate_all[ctx];
         int prev_cost = 0;
         int i, j;
-        av1_cost_tokens_from_cdf(
-            br_rate, fc->coeff_br_cdf[AOMMIN(tx_size, TX_32X32)][plane][ctx],
-            NULL);
         for (i = 0; i < COEFF_BASE_RANGE; i += BR_CDF_SIZE - 1) {
           for (j = 0; j < BR_CDF_SIZE - 1; j++) {
             pcost->lps_cost[ctx][i + j] = prev_cost + br_rate[j];
@@ -842,7 +875,8 @@ void av1_initialize_rd_consts(AV1_COMP *cpi) {
   // Frame level coefficient cost update
   if (is_frame_level_cost_upd_freq_set(cm, inter_sf->coeff_cost_upd_level,
                                        use_nonrd_pick_mode, frames_since_key))
-    av1_fill_coeff_costs(&x->coeff_costs, cm->fc, av1_num_planes(cm));
+    av1_fill_coeff_costs(&x->coeff_costs, cm->fc, av1_num_planes(cm),
+                         cpi->oxcf.mode == REALTIME);

   // Frame level mode cost update
   if (should_force_mode_cost_update(cpi) ||
diff --git a/av1/encoder/rd.h b/av1/encoder/rd.h
index f0b667f523..7326cdcd34 100644
--- a/av1/encoder/rd.h
+++ b/av1/encoder/rd.h
@@ -424,7 +424,7 @@ void av1_fill_lr_rates(ModeCosts *mode_costs, FRAME_CONTEXT *fc);
 #endif

 void av1_fill_coeff_costs(CoeffCosts *coeff_costs, FRAME_CONTEXT *fc,
-                          const int num_planes);
+                          const int num_planes, int is_realtime);

 void av1_fill_mv_costs(const nmv_context *nmvc, int integer_mv, int usehp,
                        MvCosts *mv_costs);
diff --git a/av1/encoder/x86/cost_avx2.c b/av1/encoder/x86/cost_avx2.c
new file mode 100644
index 0000000000..6a71d5e2cd
--- /dev/null
+++ b/av1/encoder/x86/cost_avx2.c
@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2026, Alliance for Open Media. All rights reserved.
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <immintrin.h>
+
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+#include "av1/encoder/cost.h"
+
+// Vector form of av1_cost_symbol_clamped for 8 lanes. Bit-exact with the
+// scalar path:
+//   p15c  = clamp(p15, EC_MIN_PROB, CDF_PROB_TOP - 1)
+//   shift = (CDF_PROB_BITS - 1) - floor(log2(p15c))   // via float exponent
+//   idx   = min(((p15c << shift) + 64) >> 7, 255) - 128
+//   cost  = av1_prob_cost[idx] + shift * 512
+// The scalar path can reach idx 128, which duplicates idx 127, so clamping to
+// 127 gives the same cost and keeps the 32-bit gather below within the table.
+static inline __m256i cost_symbol_x8(__m256i p15) {
+  p15 = _mm256_max_epi32(p15, _mm256_set1_epi32(EC_MIN_PROB));
+  p15 = _mm256_min_epi32(p15, _mm256_set1_epi32(CDF_PROB_TOP - 1));
+  // floor(log2(p15)) = (biased exponent) - 127; exact since p15 < 2^24.
+  const __m256i bits = _mm256_castps_si256(_mm256_cvtepi32_ps(p15));
+  const __m256i exp =
+      _mm256_and_si256(_mm256_srli_epi32(bits, 23), _mm256_set1_epi32(0xFF));
+  // shift = (CDF_PROB_BITS - 1) - (exp - 127) = 141 - exp.
+  const __m256i shift = _mm256_sub_epi32(_mm256_set1_epi32(141), exp);
+  const __m256i norm = _mm256_sllv_epi32(p15, shift);
+  __m256i prob =
+      _mm256_srli_epi32(_mm256_add_epi32(norm, _mm256_set1_epi32(64)), 7);
+  prob = _mm256_min_epi32(prob, _mm256_set1_epi32(255));
+  const __m256i idx = _mm256_sub_epi32(prob, _mm256_set1_epi32(128));
+  // 16-bit table read via 32-bit gather; mask off the adjacent entry (idx + 1
+  // is at most 128, which is inside the 129-entry table).
+  __m256i tbl = _mm256_i32gather_epi32((const int *)av1_prob_cost, idx, 2);
+  tbl = _mm256_and_si256(tbl, _mm256_set1_epi32(0xFFFF));
+  const __m256i lit = _mm256_slli_epi32(shift, AV1_PROB_COST_SHIFT);
+  return _mm256_add_epi32(tbl, lit);
+}
+
+void av1_cost_tokens_from_cdf_4_batch_avx2(int *costs, int costs_stride,
+                                           const uint16_t *cdf, int cdf_stride,
+                                           int n) {
+  const int s = cdf_stride;
+  // Per-lane element offset of the 8 CDFs processed together.
+  const __m256i lane =
+      _mm256_setr_epi32(0, s, 2 * s, 3 * s, 4 * s, 5 * s, 6 * s, 7 * s);
+  const __m256i mask16 = _mm256_set1_epi32(0xFFFF);
+  const __m256i top = _mm256_set1_epi32(CDF_PROB_TOP);
+
+  int c = 0;
+  for (; c + 8 <= n; c += 8) {
+    const __m256i base = _mm256_add_epi32(_mm256_set1_epi32(c * s), lane);
+    // Gather {c0,c1} and {c2,c3} pairs (16-bit values packed in 32-bit lanes).
+    const __m256i g01 = _mm256_i32gather_epi32((const int *)cdf, base, 2);
+    const __m256i g23 = _mm256_i32gather_epi32(
+        (const int *)cdf, _mm256_add_epi32(base, _mm256_set1_epi32(2)), 2);
+    // ICDF(x) = CDF_PROB_TOP - x; p15[i] = ICDF[i] - ICDF[i-1], ICDF[-1] = 0.
+    const __m256i i0 = _mm256_sub_epi32(top, _mm256_and_si256(g01, mask16));
+    const __m256i i1 = _mm256_sub_epi32(top, _mm256_srli_epi32(g01, 16));
+    const __m256i i2 = _mm256_sub_epi32(top, _mm256_and_si256(g23, mask16));
+    const __m256i i3 = _mm256_sub_epi32(top, _mm256_srli_epi32(g23, 16));
+    const __m256i r0 = cost_symbol_x8(i0);
+    const __m256i r1 = cost_symbol_x8(_mm256_sub_epi32(i1, i0));
+    const __m256i r2 = cost_symbol_x8(_mm256_sub_epi32(i2, i1));
+    const __m256i r3 = cost_symbol_x8(_mm256_sub_epi32(i3, i2));
+
+    // Transpose the 4 SoA cost vectors (8 ctx each) into 8 AoS rows of 4.
+    const __m256i t0 = _mm256_unpacklo_epi32(r0, r1);
+    const __m256i t1 = _mm256_unpackhi_epi32(r0, r1);
+    const __m256i t2 = _mm256_unpacklo_epi32(r2, r3);
+    const __m256i t3 = _mm256_unpackhi_epi32(r2, r3);
+    const __m256i q0 = _mm256_unpacklo_epi64(t0, t2);  // ctx 0 | ctx 4
+    const __m256i q1 = _mm256_unpackhi_epi64(t0, t2);  // ctx 1 | ctx 5
+    const __m256i q2 = _mm256_unpacklo_epi64(t1, t3);  // ctx 2 | ctx 6
+    const __m256i q3 = _mm256_unpackhi_epi64(t1, t3);  // ctx 3 | ctx 7
+
+    int *o = costs + c * costs_stride;
+    _mm_storeu_si128((__m128i *)(o + 0 * costs_stride),
+                     _mm256_castsi256_si128(q0));
+    _mm_storeu_si128((__m128i *)(o + 1 * costs_stride),
+                     _mm256_castsi256_si128(q1));
+    _mm_storeu_si128((__m128i *)(o + 2 * costs_stride),
+                     _mm256_castsi256_si128(q2));
+    _mm_storeu_si128((__m128i *)(o + 3 * costs_stride),
+                     _mm256_castsi256_si128(q3));
+    _mm_storeu_si128((__m128i *)(o + 4 * costs_stride),
+                     _mm256_extracti128_si256(q0, 1));
+    _mm_storeu_si128((__m128i *)(o + 5 * costs_stride),
+                     _mm256_extracti128_si256(q1, 1));
+    _mm_storeu_si128((__m128i *)(o + 6 * costs_stride),
+                     _mm256_extracti128_si256(q2, 1));
+    _mm_storeu_si128((__m128i *)(o + 7 * costs_stride),
+                     _mm256_extracti128_si256(q3, 1));
+  }
+  for (; c < n; ++c)
+    av1_cost_tokens_from_cdf_4(costs + c * costs_stride, cdf + c * cdf_stride);
+}
+
+void av1_cost_tokens_from_cdf_2_batch_avx2(int *costs, int costs_stride,
+                                           const uint16_t *cdf, int cdf_stride,
+                                           int n) {
+  const int s = cdf_stride;
+  const __m256i lane =
+      _mm256_setr_epi32(0, s, 2 * s, 3 * s, 4 * s, 5 * s, 6 * s, 7 * s);
+  const __m256i top = _mm256_set1_epi32(CDF_PROB_TOP);
+
+  int c = 0;
+  for (; c + 8 <= n; c += 8) {
+    const __m256i base = _mm256_add_epi32(_mm256_set1_epi32(c * s), lane);
+    // Only cdf[0] is needed for a 2-symbol CDF; the 32-bit read's high half is
+    // the terminator and is masked away.
+    const __m256i g = _mm256_i32gather_epi32((const int *)cdf, base, 2);
+    const __m256i c0 = _mm256_and_si256(g, _mm256_set1_epi32(0xFFFF));
+    // p15[0] = ICDF[0] = CDF_PROB_TOP - cdf[0]; p15[1] = cdf[0].
+    const __m256i r0 = cost_symbol_x8(_mm256_sub_epi32(top, c0));
+    const __m256i r1 = cost_symbol_x8(c0);
+
+    // Transpose the 2 SoA cost vectors into 8 AoS pairs and store 2 ints/ctx.
+    const __m256i t0 = _mm256_unpacklo_epi32(r0, r1);  // ctx 0,1 | ctx 4,5
+    const __m256i t1 = _mm256_unpackhi_epi32(r0, r1);  // ctx 2,3 | ctx 6,7
+    const __m128i lo0 = _mm256_castsi256_si128(t0);
+    const __m128i hi0 = _mm256_extracti128_si256(t0, 1);
+    const __m128i lo1 = _mm256_castsi256_si128(t1);
+    const __m128i hi1 = _mm256_extracti128_si256(t1, 1);
+    int *o = costs + c * costs_stride;
+    _mm_storel_epi64((__m128i *)(o + 0 * costs_stride), lo0);
+    _mm_storel_epi64((__m128i *)(o + 1 * costs_stride),
+                     _mm_unpackhi_epi64(lo0, lo0));
+    _mm_storel_epi64((__m128i *)(o + 2 * costs_stride), lo1);
+    _mm_storel_epi64((__m128i *)(o + 3 * costs_stride),
+                     _mm_unpackhi_epi64(lo1, lo1));
+    _mm_storel_epi64((__m128i *)(o + 4 * costs_stride), hi0);
+    _mm_storel_epi64((__m128i *)(o + 5 * costs_stride),
+                     _mm_unpackhi_epi64(hi0, hi0));
+    _mm_storel_epi64((__m128i *)(o + 6 * costs_stride), hi1);
+    _mm_storel_epi64((__m128i *)(o + 7 * costs_stride),
+                     _mm_unpackhi_epi64(hi1, hi1));
+  }
+  for (; c < n; ++c)
+    av1_cost_tokens_from_cdf_2(costs + c * costs_stride, cdf + c * cdf_stride);
+}
diff --git a/test/cost_test.cc b/test/cost_test.cc
new file mode 100644
index 0000000000..c297a3babf
--- /dev/null
+++ b/test/cost_test.cc
@@ -0,0 +1,225 @@
+/*
+ * Copyright (c) 2026, Alliance for Open Media. All rights reserved.
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <algorithm>
+#include <vector>
+
+#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+#include "aom_dsp/prob.h"
+#include "aom_ports/aom_timer.h"
+#include "av1/common/entropy.h"
+#include "av1/encoder/cost.h"
+#include "test/acm_random.h"
+#include "gtest/gtest.h"
+
+namespace {
+
+using libaom_test::ACMRandom;
+
+using Cdf4BatchFn = void (*)(int *costs, int costs_stride, const uint16_t *cdf,
+                             int cdf_stride, int n);
+
+// Fixture that fills av1_cost_lut_low once before exercising the cost helpers.
+class CostTokensFromCdf4Test : public ::testing::Test {
+ protected:
+  void SetUp() override { av1_init_cost_luts(); }
+};
+
+// Builds a valid stored 4-symbol CDF from monotonic cumulative probs.
+void BuildCdf4(ACMRandom *rnd, aom_cdf_prob cdf[5]) {
+  uint16_t v[3];
+  for (int k = 0; k < 3; ++k) v[k] = 1 + (rnd->Rand16() % (CDF_PROB_TOP - 1));
+  // Keep it monotonic; leave duplicates in so the EC_MIN_PROB clamp gets
+  // exercised.
+  std::sort(v, v + 3);
+  cdf[0] = AOM_ICDF(v[0]);
+  cdf[1] = AOM_ICDF(v[1]);
+  cdf[2] = AOM_ICDF(v[2]);
+  cdf[3] = AOM_ICDF(CDF_PROB_TOP);
+  cdf[4] = 0;  // CDF adaptation counter slot; not read by the cost path.
+}
+
+// Builds a valid stored 2-symbol CDF.
+void BuildCdf2(ACMRandom *rnd, aom_cdf_prob cdf[3]) {
+  cdf[0] = AOM_ICDF(1 + (rnd->Rand16() % (CDF_PROB_TOP - 1)));
+  cdf[1] = AOM_ICDF(CDF_PROB_TOP);
+  cdf[2] = 0;  // CDF adaptation counter slot; not read by the cost path.
+}
+
+// The hybrid LUT+arithmetic path must equal the pure-arithmetic
+// av1_cost_symbol for every p15, including the COST_LUT_LOW boundary and the
+// fallback range.
+TEST_F(CostTokensFromCdf4Test, HybridMatchesArithmetic) {
+  for (int p = 1; p < CDF_PROB_TOP; ++p)
+    ASSERT_EQ(av1_cost_symbol((aom_cdf_prob)p), av1_cost_symbol_lut(p))
+        << "p15=" << p;
+}
+
+// The N=4 specialization must be bit-identical to the generic path.
+TEST_F(CostTokensFromCdf4Test, MatchesGenericRandom) {
+  ACMRandom rnd(ACMRandom::DeterministicSeed());
+  for (int iter = 0; iter < 200000; ++iter) {
+    aom_cdf_prob cdf[5];
+    BuildCdf4(&rnd, cdf);
+    int ref[4], spec[4];
+    av1_cost_tokens_from_cdf(ref, cdf, nullptr);
+    av1_cost_tokens_from_cdf_4(spec, cdf);
+    for (int i = 0; i < 4; ++i)
+      ASSERT_EQ(ref[i], spec[i]) << "iter=" << iter << " i=" << i;
+  }
+}
+
+// Degenerate CDFs (near-zero probs) must clamp identically.
+TEST_F(CostTokensFromCdf4Test, MatchesGenericEdge) {
+  const aom_cdf_prob edge[][5] = {
+    { AOM_ICDF(1), AOM_ICDF(2), AOM_ICDF(3), AOM_ICDF(CDF_PROB_TOP), 0 },
+    { AOM_ICDF(1), AOM_ICDF(1), AOM_ICDF(1), AOM_ICDF(CDF_PROB_TOP), 0 },
+    { AOM_ICDF(CDF_PROB_TOP - 3), AOM_ICDF(CDF_PROB_TOP - 2),
+      AOM_ICDF(CDF_PROB_TOP - 1), AOM_ICDF(CDF_PROB_TOP), 0 },
+    { AOM_ICDF(8192), AOM_ICDF(16384), AOM_ICDF(24576), AOM_ICDF(CDF_PROB_TOP),
+      0 },
+  };
+  for (const auto &cdf : edge) {
+    int ref[4], spec[4];
+    av1_cost_tokens_from_cdf(ref, cdf, nullptr);
+    av1_cost_tokens_from_cdf_4(spec, cdf);
+    for (int i = 0; i < 4; ++i) ASSERT_EQ(ref[i], spec[i]) << "i=" << i;
+  }
+}
+
+// Runs `fn` for a range of context counts and output strides and checks each
+// filled entry against the generic scalar path. Exercises the SIMD main loop
+// plus every tail length (n % 8) and both output strides (8 = base_cost,
+// 4 = br_rate).
+void CheckBatchAgainstGeneric(Cdf4BatchFn fn) {
+  ACMRandom rnd(ACMRandom::DeterministicSeed());
+  const int kCdfStride = CDF_SIZE(4);  // 5
+  for (const int costs_stride : { 4, 8 }) {
+    for (int n = 1; n <= 45; ++n) {
+      std::vector<aom_cdf_prob> cdf(n * kCdfStride);
+      for (int k = 0; k < n; ++k) BuildCdf4(&rnd, &cdf[k * kCdfStride]);
+
+      std::vector<int> out(n * costs_stride, -1);
+      std::vector<int> ref(n * 4);
+      for (int k = 0; k < n; ++k)
+        av1_cost_tokens_from_cdf(&ref[k * 4], &cdf[k * kCdfStride], nullptr);
+
+      fn(out.data(), costs_stride, cdf.data(), kCdfStride, n);
+
+      for (int k = 0; k < n; ++k)
+        for (int i = 0; i < 4; ++i)
+          ASSERT_EQ(ref[k * 4 + i], out[k * costs_stride + i])
+              << "stride=" << costs_stride << " n=" << n << " k=" << k
+              << " i=" << i;
+    }
+  }
+}
+
+TEST_F(CostTokensFromCdf4Test, BatchCMatchesGeneric) {
+  CheckBatchAgainstGeneric(av1_cost_tokens_from_cdf_4_batch_c);
+}
+
+#if HAVE_AVX2
+TEST_F(CostTokensFromCdf4Test, BatchAVX2MatchesGeneric) {
+  CheckBatchAgainstGeneric(av1_cost_tokens_from_cdf_4_batch_avx2);
+}
+#endif  // HAVE_AVX2
+
+// Same as CheckBatchAgainstGeneric but for the N=2 batch primitive.
+void CheckBatch2AgainstGeneric(Cdf4BatchFn fn) {
+  ACMRandom rnd(ACMRandom::DeterministicSeed());
+  const int kCdfStride = CDF_SIZE(2);  // 3
+  for (const int costs_stride : { 2, 4 }) {
+    for (int n = 1; n <= 45; ++n) {
+      std::vector<aom_cdf_prob> cdf(n * kCdfStride);
+      for (int k = 0; k < n; ++k) BuildCdf2(&rnd, &cdf[k * kCdfStride]);
+
+      std::vector<int> out(n * costs_stride, -1);
+      std::vector<int> ref(n * 2);
+      for (int k = 0; k < n; ++k)
+        av1_cost_tokens_from_cdf(&ref[k * 2], &cdf[k * kCdfStride], nullptr);
+
+      fn(out.data(), costs_stride, cdf.data(), kCdfStride, n);
+
+      for (int k = 0; k < n; ++k)
+        for (int i = 0; i < 2; ++i)
+          ASSERT_EQ(ref[k * 2 + i], out[k * costs_stride + i])
+              << "stride=" << costs_stride << " n=" << n << " k=" << k
+              << " i=" << i;
+    }
+  }
+}
+
+TEST_F(CostTokensFromCdf4Test, Batch2CMatchesGeneric) {
+  CheckBatch2AgainstGeneric(av1_cost_tokens_from_cdf_2_batch_c);
+}
+
+#if HAVE_AVX2
+TEST_F(CostTokensFromCdf4Test, Batch2AVX2MatchesGeneric) {
+  CheckBatch2AgainstGeneric(av1_cost_tokens_from_cdf_2_batch_avx2);
+}
+#endif  // HAVE_AVX2
+
+#if HAVE_AVX2
+// Times the C vs AVX2 batch primitive on the real av1_fill_coeff_costs call
+// shapes and prints ns/call and the speedup. DISABLED_ so it runs only on
+// request (--gtest_also_run_disabled_tests).
+void RunBatchSpeed(const char *name, Cdf4BatchFn c_fn, Cdf4BatchFn simd_fn,
+                   int cdf_stride, int syms, const char *label, int n,
+                   int costs_stride) {
+  ACMRandom rnd(ACMRandom::DeterministicSeed());
+  std::vector<aom_cdf_prob> cdf(n * cdf_stride);
+  for (int k = 0; k < n; ++k) {
+    if (syms == 4)
+      BuildCdf4(&rnd, &cdf[k * cdf_stride]);
+    else
+      BuildCdf2(&rnd, &cdf[k * cdf_stride]);
+  }
+  std::vector<int> out(n * costs_stride);
+
+  const Cdf4BatchFn funcs[2] = { c_fn, simd_fn };
+  const int kLoops = 1000000;
+  double ns[2];
+  for (int i = 0; i < 2; ++i) {
+    aom_usec_timer timer;
+    aom_usec_timer_start(&timer);
+    for (int j = 0; j < kLoops; ++j)
+      funcs[i](out.data(), costs_stride, cdf.data(), cdf_stride, n);
+    aom_usec_timer_mark(&timer);
+    ns[i] =
+        1000.0 * static_cast<double>(aom_usec_timer_elapsed(&timer)) / kLoops;
+  }
+  printf("%s %-9s n=%2d stride=%d: c %7.1f ns  avx2 %7.1f ns  (%4.2fx)\n", name,
+         label, n, costs_stride, ns[0], ns[1], ns[0] / ns[1]);
+}
+
+TEST_F(CostTokensFromCdf4Test, DISABLED_Batch4Speed) {
+  RunBatchSpeed("cdf_4_batch", av1_cost_tokens_from_cdf_4_batch_c,
+                av1_cost_tokens_from_cdf_4_batch_avx2, CDF_SIZE(4), 4,
+                "base_cost", SIG_COEF_CONTEXTS, 8);
+  RunBatchSpeed("cdf_4_batch", av1_cost_tokens_from_cdf_4_batch_c,
+                av1_cost_tokens_from_cdf_4_batch_avx2, CDF_SIZE(4), 4,
+                "br_rate", LEVEL_CONTEXTS, BR_CDF_SIZE);
+}
+
+TEST_F(CostTokensFromCdf4Test, DISABLED_Batch2Speed) {
+  RunBatchSpeed("cdf_2_batch", av1_cost_tokens_from_cdf_2_batch_c,
+                av1_cost_tokens_from_cdf_2_batch_avx2, CDF_SIZE(2), 2,
+                "txb_skip", TXB_SKIP_CONTEXTS, 2);
+  RunBatchSpeed("cdf_2_batch", av1_cost_tokens_from_cdf_2_batch_c,
+                av1_cost_tokens_from_cdf_2_batch_avx2, CDF_SIZE(2), 2,
+                "eob_extra", EOB_COEF_CONTEXTS, 2);
+}
+#endif  // HAVE_AVX2
+
+}  // namespace
diff --git a/test/test.cmake b/test/test.cmake
index db48c730af..b21e1f8e5f 100644
--- a/test/test.cmake
+++ b/test/test.cmake
@@ -219,6 +219,7 @@ if(NOT BUILD_SHARED_LIBS)
               "${AOM_ROOT}/test/blend_a64_mask_1d_test.cc"
               "${AOM_ROOT}/test/blend_a64_mask_test.cc"
               "${AOM_ROOT}/test/comp_mask_pred_test.cc"
+              "${AOM_ROOT}/test/cost_test.cc"
               "${AOM_ROOT}/test/disflow_test.cc"
               "${AOM_ROOT}/test/encode_internal_test.cc"
               "${AOM_ROOT}/test/encodemb_test.cc"