Commit 03afa68a21 for aom

commit 03afa68a2191752320db67e27d07115a5c5a4c7a
Author: Lin Zheng <linzhen@google.com>
Date:   Thu Sep 17 21:29:11 2026 +0000

    Smooth 2-pass rate control response and bit redistribution

    Allow over_shoot_pct <= 100 and unclamp pct_error for HBD sharpness 3
    mode to adapt more smoothly to high bitrate peaks.
    Suppress fast extra bit redistribution when currently overshooting
    to prevent inter-frame QP oscillations.

    Bug: 548701298
    Change-Id: Iacdd96a988ea3ad97eae89a32400659d8f3e5794

diff --git a/av1/encoder/pass2_strategy.c b/av1/encoder/pass2_strategy.c
index 36775ffe16..42e9186f06 100644
--- a/av1/encoder/pass2_strategy.c
+++ b/av1/encoder/pass2_strategy.c
@@ -4600,13 +4600,12 @@ void av1_twopass_postencode_update(AV1_COMP *cpi) {
       }
       twopass->extend_maxq -= 1;
       // Overshoot
-    } else if ((rc_cfg->over_shoot_pct < 100) &&
+    } else if ((rc_cfg->over_shoot_pct <= 100) &&
                (p_rc->rolling_actual_bits > p_rc->rolling_target_bits)) {
       int pct_error =
           ((p_rc->rolling_actual_bits - p_rc->rolling_target_bits) * 100) /
           p_rc->rolling_target_bits;

-      pct_error = clamp(pct_error, 0, 100);
       if ((pct_error >= rc_cfg->over_shoot_pct) &&
           (p_rc->rate_error_estimate < 0)) {
         twopass->extend_maxq += 1;
@@ -4632,13 +4631,21 @@ void av1_twopass_postencode_update(AV1_COMP *cpi) {
     // frame is unexpectedly almost perfectly predicted by the ARF or GF
     // but not very well predcited by the previous frame.
     if (!frame_is_kf_gf_arf(cpi) && !cpi->rc.is_src_frame_alt_ref) {
-      int fast_extra_thresh = rc->base_frame_target / HIGH_UNDERSHOOT_RATIO;
-      if (rc->projected_frame_size < fast_extra_thresh) {
-        p_rc->vbr_bits_off_target_fast +=
-            fast_extra_thresh - rc->projected_frame_size;
-        p_rc->vbr_bits_off_target_fast =
-            AOMMIN(p_rc->vbr_bits_off_target_fast,
-                   (4 * (int64_t)rc->avg_frame_bandwidth));
+#if CONFIG_AV1_HIGHBITDEPTH
+      if (cpi->common.seq_params->bit_depth > 8 &&
+          cpi->oxcf.algo_cfg.sharpness == 3 && p_rc->vbr_bits_off_target < 0) {
+        p_rc->vbr_bits_off_target_fast = 0;
+      } else
+#endif
+      {
+        int fast_extra_thresh = rc->base_frame_target / HIGH_UNDERSHOOT_RATIO;
+        if (rc->projected_frame_size < fast_extra_thresh) {
+          p_rc->vbr_bits_off_target_fast +=
+              fast_extra_thresh - rc->projected_frame_size;
+          p_rc->vbr_bits_off_target_fast =
+              AOMMIN(p_rc->vbr_bits_off_target_fast,
+                     (4 * (int64_t)rc->avg_frame_bandwidth));
+        }
       }
     }

diff --git a/av1/encoder/ratectrl.c b/av1/encoder/ratectrl.c
index 677432fb9d..03cb5c945d 100644
--- a/av1/encoder/ratectrl.c
+++ b/av1/encoder/ratectrl.c
@@ -2789,13 +2789,17 @@ static void vbr_rate_correction(AV1_COMP *cpi, int *this_frame_target) {
       simulate_parallel_frame ? cpi->ppi->p_rc.temp_vbr_bits_off_target_fast
                               : p_rc->vbr_bits_off_target_fast;
 #endif
-  // Fast redistribution of bits arising from massive local undershoot.
-  // Don't do it for kf,arf,gf or overlay frames.
   if (!frame_is_kf_gf_arf(cpi) &&
 #if CONFIG_FPMT_TEST
       vbr_bits_off_target_fast &&
 #else
       p_rc->vbr_bits_off_target_fast &&
+#endif
+#if CONFIG_AV1_HIGHBITDEPTH
+      (cpi->common.seq_params->bit_depth > 8 &&
+               cpi->oxcf.algo_cfg.sharpness == 3
+           ? vbr_bits_off_target >= 0
+           : 1) &&
 #endif
       !rc->is_src_frame_alt_ref) {
     int64_t one_frame_bits = AOMMAX(rc->avg_frame_bandwidth, frame_target);