Commit 9517176b87 for aom

commit 9517176b8702ff1eae2077041c844c4d59dd8371
Author: Frank Galligan <fgalligan@google.com>
Date:   Fri Oct 30 12:07:14 2026 -0700

    av1: Optimize avg_cdf_symbol and AC dequant

    - Specialize avg_cdf_symbol for the common (wt_left == 3, wt_tr == 1)
      row-multithreading CDF update using shift-and-average arithmetic that
      auto-vectorizes with 16-bit SIMD instructions.
    - Use dequant[1] directly in update_coeff_simple since it is only
      called on AC coefficients (si > 0).

    Benchmarking results (incremental vs prev CL / cumulative vs baseline):
    - Stock avifenc still-image (Default 1T): -0.45% / -18.78% encode time
    - Stock avifenc still-image (Default 8T): -0.50% / -15.05% encode time
    - 48-image multi-rate corpus (Full Suite 1T): -0.55% / -18.73% time
    - Bit-exact output (1T and 8T)

    TAG=agy
    CONV=b929a630-a6cd-45a0-979c-1767c30178ee
    Change-Id: Ib997b3791b37da2622a0237c9ac0c51e0b7af90b

diff --git a/av1/encoder/encodeframe_utils.c b/av1/encoder/encodeframe_utils.c
index 0dc7ef74b2..58cb0ea067 100644
--- a/av1/encoder/encodeframe_utils.c
+++ b/av1/encoder/encodeframe_utils.c
@@ -11,6 +11,7 @@

 #include "aom/aom_codec.h"
 #include "aom/internal/aom_codec_internal.h"
+#include <stdlib.h>

 #include "av1/common/common_data.h"
 #include "av1/common/quant_common.h"
@@ -1072,9 +1073,13 @@ void av1_update_picked_ref_frames_mask(MACROBLOCK *const x, int ref_type,
   }
 }

-static void avg_cdf_symbol(aom_cdf_prob *cdf_ptr_left, aom_cdf_prob *cdf_ptr_tr,
-                           int num_cdfs, int cdf_stride, int nsymbs,
-                           int wt_left, int wt_tr) {
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
+static void avg_cdf_symbol_general(aom_cdf_prob *cdf_ptr_left,
+                                   const aom_cdf_prob *cdf_ptr_tr, int num_cdfs,
+                                   int cdf_stride, int nsymbs, int wt_left,
+                                   int wt_tr) {
   for (int i = 0; i < num_cdfs; i++) {
     for (int j = 0; j <= nsymbs; j++) {
       cdf_ptr_left[i * cdf_stride + j] =
@@ -1088,6 +1093,38 @@ static void avg_cdf_symbol(aom_cdf_prob *cdf_ptr_left, aom_cdf_prob *cdf_ptr_tr,
   }
 }

+static inline void avg_cdf_symbol(aom_cdf_prob *restrict cdf_ptr_left,
+                                  const aom_cdf_prob *restrict cdf_ptr_tr,
+                                  int num_cdfs, int cdf_stride, int nsymbs,
+                                  int wt_left, int wt_tr) {
+  // Fast path for the common 3:1 row-mt CDF weighting:
+  // (3 * a + b + 2) >> 2 == (a + ((a + b) >> 1) + 1) >> 1, which compilers
+  // auto-vectorize using unsigned 16-bit average instructions.
+  if (wt_left == 3 && wt_tr == 1) {
+    if (cdf_stride == nsymbs + 1) {
+      const int total = num_cdfs * cdf_stride;
+      for (int idx = 0; idx < total; idx++) {
+        const uint16_t a = cdf_ptr_left[idx];
+        const uint16_t b = cdf_ptr_tr[idx];
+        cdf_ptr_left[idx] =
+            (aom_cdf_prob)((a + ((uint16_t)(a + b) >> 1) + 1) >> 1);
+      }
+    } else {
+      for (int i = 0; i < num_cdfs; i++) {
+        for (int j = 0; j <= nsymbs; j++) {
+          const uint16_t a = cdf_ptr_left[i * cdf_stride + j];
+          const uint16_t b = cdf_ptr_tr[i * cdf_stride + j];
+          cdf_ptr_left[i * cdf_stride + j] =
+              (aom_cdf_prob)((a + ((uint16_t)(a + b) >> 1) + 1) >> 1);
+        }
+      }
+    }
+  } else {
+    avg_cdf_symbol_general(cdf_ptr_left, cdf_ptr_tr, num_cdfs, cdf_stride,
+                           nsymbs, wt_left, wt_tr);
+  }
+}
+
 #define AVERAGE_CDF(cname_left, cname_tr, nsymbs) \
   AVG_CDF_STRIDE(cname_left, cname_tr, nsymbs, CDF_SIZE(nsymbs))

diff --git a/av1/encoder/txb_rdopt.c b/av1/encoder/txb_rdopt.c
index f032171651..4b09adea62 100644
--- a/av1/encoder/txb_rdopt.c
+++ b/av1/encoder/txb_rdopt.c
@@ -151,7 +151,12 @@ static AOM_FORCE_INLINE int update_coeff_simple(
       int rate_low = 0;
       const int rate = get_two_coeff_cost_simple(
           ci, abs_qc, coeff_ctx, txb_costs, bhl, tx_class, levels, &rate_low);
-      const int dqv = get_dqv(dequant, ci, iqmatrix);
+      // update_coeff_simple is only invoked for AC coefficients (si > 0), so
+      // dequant[1] can be used directly without a (ci > 0) branch.
+      int dqv = dequant[1];
+      if (iqmatrix != NULL) {
+        dqv = ((iqmatrix[ci] * dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
+      }
       const tran_low_t abs_qc_low = abs_qc - 1;
       const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift;
       int64_t dist_diff_0, dist_diff_low_0;