Commit d03d2c63a1 for aom

commit d03d2c63a132391bfa219d421c492b67aee91105
Author: Erik Språng <sprang@google.com>
Date:   Sun Nov 15 18:00:28 2026 +0100

    rtc: Incorrect PSNR for non-reference frames

    In real-time mode, derive_skip_apply_postproc_filters() unconditionally
    returned SKIP_APPLY_LOOPFILTER | SKIP_APPLY_CDEF for non-reference
    frames (rtc_ref.non_reference_frame == 1), even when PSNR calculation
    was requested via AOM_EFLAG_CALCULATE_PSNR or AOM_CODEC_USE_PSNR. As a
    result, AOM_CODEC_PSNR_PKT reported SSE/PSNR against the unfiltered
    reconstruction buffer rather than the actual decoded output.

    Check cpi->ppi->b_calculate_psnr at the start of
    derive_skip_apply_postproc_filters() so post-processing filters are
    applied whenever PSNR calculation is active, and update the
    corresponding assertions in encoder.c and pipelined loop-filter
    initialization in ethread.c.

    Note that for cdef_pick_method <= CDEF_FAST_SEARCH_LVL5, applying the
    loop filter on non-reference frames when PSNR is requested means CDEF
    strength search runs on deblocked pixels (matching reference frames and
    decoder order), which can slightly change CDEF parameter selection
    compared to encoding without PSNR calculation.

    Change-Id: Ic2f540776f21e3ada3453f2225f9cc6073fe02bc
    Bug: 586156919, webrtc:496266459

diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index 15aed5415c..10167023ba 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -2836,7 +2836,8 @@ static void cdef_restoration_frame(AV1_COMP *cpi, AV1_COMMON *cm,

     // Apply the filter
     if ((skip_apply_postproc_filters & SKIP_APPLY_CDEF) == 0) {
-      assert(!cpi->ppi->rtc_ref.non_reference_frame);
+      assert(!cpi->ppi->rtc_ref.non_reference_frame ||
+             cpi->ppi->b_calculate_psnr);
       if (num_workers > 1) {
         // Extension of frame borders is multi-threaded along with cdef.
         const int do_extend_border =
@@ -2940,7 +2941,8 @@ static void loopfilter_frame(AV1_COMP *cpi, AV1_COMMON *cm) {
     struct loopfilter *lf = &cm->lf;
     if ((lf->filter_level[0] || lf->filter_level[1]) &&
         (skip_apply_postproc_filters & SKIP_APPLY_LOOPFILTER) == 0) {
-      assert(!cpi->ppi->rtc_ref.non_reference_frame);
+      assert(!cpi->ppi->rtc_ref.non_reference_frame ||
+             cpi->ppi->b_calculate_psnr);
       // lpf_opt_level = 1 : Enables dual/quad loop-filtering.
       // lpf_opt_level is set to 1 if transform size search depth in inter
       // blocks is limited to one as quad loop filtering assumes that all the
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index f6096dac03..a5201682a1 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -4518,16 +4518,20 @@ static inline int is_restoration_used(const AV1_COMMON *const cm) {
 static inline unsigned int derive_skip_apply_postproc_filters(
     const AV1_COMP *cpi, int use_loopfilter, int use_cdef, int use_superres,
     int use_restoration) {
+  // Don't skip CDEF or loop filter when PSNR is enabled, since the comparison
+  // against the reconstructed state would then diverge from the ground truth.
+  if (cpi->ppi->b_calculate_psnr) return 0;
+
   // Though CDEF parameter selection should be dependent on
   // deblocked/loop-filtered pixels for cdef_pick_method <=
   // CDEF_FAST_SEARCH_LVL5, CDEF strength values are calculated based on the
-  // pixel values that are not loop-filtered in svc real-time encoding mode.
+  // pixel values that are not loop-filtered in svc real-time encoding mode
+  // when PSNR calculation is disabled.
   // Hence this case is handled separately using the condition below.
   if (cpi->ppi->rtc_ref.non_reference_frame)
     return (SKIP_APPLY_LOOPFILTER | SKIP_APPLY_CDEF);

-  if (!cpi->oxcf.algo_cfg.skip_postproc_filtering || cpi->ppi->b_calculate_psnr)
-    return 0;
+  if (!cpi->oxcf.algo_cfg.skip_postproc_filtering) return 0;
   assert(cpi->oxcf.mode == ALLINTRA);

   // The post-processing filters are applied one after the other in the
diff --git a/av1/encoder/ethread.c b/av1/encoder/ethread.c
index 1690e21878..8ca1efc38c 100644
--- a/av1/encoder/ethread.c
+++ b/av1/encoder/ethread.c
@@ -1869,8 +1869,9 @@ static void lpf_pipeline_mt_init(AV1_COMP *cpi, int num_workers) {
   // Pipelining of loop-filtering after encoding is enabled when loop-filter
   // level is chosen based on quantizer and frame type. It is disabled in case
   // of 'LOOPFILTER_SELECTIVELY' as the stats collected during encoding stage
-  // decides the filter level. Loop-filtering is disabled in case
-  // of non-reference frames and for frames with intra block copy tool enabled.
+  // decides the filter level. Loop-filtering is disabled when
+  // SKIP_APPLY_LOOPFILTER is set and for frames with intra block copy tool
+  // enabled.
   AV1_COMMON *cm = &cpi->common;
   const int use_loopfilter = is_loopfilter_used(cm);
   const int use_superres = av1_superres_scaled(cm);
@@ -1886,7 +1887,7 @@ static void lpf_pipeline_mt_init(AV1_COMP *cpi, int num_workers) {
       (cpi->oxcf.mode == REALTIME) && (cpi->oxcf.speed >= 5) &&
       (cpi->sf.lpf_sf.lpf_pick == LPF_PICK_FROM_Q) &&
       (cpi->oxcf.algo_cfg.loopfilter_control != LOOPFILTER_SELECTIVELY) &&
-      !cpi->ppi->rtc_ref.non_reference_frame && !cm->features.allow_intrabc &&
+      !cm->features.allow_intrabc &&
       ((skip_apply_postproc_filters & SKIP_APPLY_LOOPFILTER) == 0);

   if (!mt_info->pipeline_lpf_mt_with_enc) return;
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index ea42365a7a..edfa15773c 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -9,6 +9,7 @@
  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
  */

+#include <algorithm>
 #include <array>
 #include <cassert>
 #include <climits>
@@ -2448,6 +2449,173 @@ TEST(EncodeAPI, PerFramePsnr) {
   ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
 }

+#if CONFIG_AV1_DECODER
+using PsnrPkt = decltype(aom_codec_cx_pkt_t().data.psnr);
+
+// Computes 8-bit I420 PSNR stats (total/Y/U/V) between two images of identical
+// dimensions, matching the layout of aom_codec_cx_pkt_t::data.psnr.
+PsnrPkt ComputeI420Psnr(const aom_image_t *img1, const aom_image_t *img2) {
+  assert(img1->fmt == AOM_IMG_FMT_I420 && img2->fmt == AOM_IMG_FMT_I420);
+  assert(img1->d_w == img2->d_w && img1->d_h == img2->d_h);
+
+  PsnrPkt stats = {};
+  for (int plane = 0; plane < 3; ++plane) {
+    const int ss = plane ? 1 : 0;
+    const unsigned int w = (img1->d_w + ss) >> ss;
+    const unsigned int h = (img1->d_h + ss) >> ss;
+    stats.samples[plane + 1] = w * h;
+    stats.samples[0] += stats.samples[plane + 1];
+    for (unsigned int y = 0; y < h; ++y) {
+      const uint8_t *row1 = img1->planes[plane] + y * img1->stride[plane];
+      const uint8_t *row2 = img2->planes[plane] + y * img2->stride[plane];
+      for (unsigned int x = 0; x < w; ++x) {
+        const int diff = static_cast<int>(row1[x]) - static_cast<int>(row2[x]);
+        stats.sse[plane + 1] += static_cast<uint64_t>(diff * diff);
+      }
+    }
+    stats.sse[0] += stats.sse[plane + 1];
+  }
+  constexpr double kMaxPsnr = 100.0;
+  constexpr double kPeak = 255.0;
+  for (int c = 0; c < 4; ++c) {
+    if (stats.sse[c] == 0) {
+      stats.psnr[c] = kMaxPsnr;
+    } else {
+      const double psnr = 10.0 * std::log10(stats.samples[c] * kPeak * kPeak /
+                                            static_cast<double>(stats.sse[c]));
+      stats.psnr[c] = std::min(psnr, kMaxPsnr);
+    }
+  }
+  return stats;
+}
+
+TEST(EncodeAPI, PerFramePsnrNonRefTemporalLayer) {
+  constexpr int kWidth = 320;
+  constexpr int kHeight = 192;
+  constexpr int kFrames = 4;
+  aom_codec_iface_t *iface = aom_codec_av1_cx();
+
+  struct TestConfig {
+    int speed;
+    int threads;
+  };
+  // Cover both realtime pick modes, CDEF pick methods, and loop-filter paths:
+  // - speed 6, 1 thread: RD pick mode with search-based CDEF
+  //   (CDEF_FAST_SEARCH_LVL4) and post-encode loopfilter_frame() in encoder.c.
+  // - speed 10, 2 threads: non-RD pick mode with quantizer-based CDEF
+  //   (CDEF_PICK_FROM_Q) and row-MT pipelined loop filtering
+  //   (lpf_pipeline_mt_init()) in ethread.c.
+  constexpr TestConfig kConfigs[] = {
+    { /*speed=*/6, /*threads=*/1 },
+    { /*speed=*/10, /*threads=*/2 },
+  };
+
+  for (const TestConfig &test_cfg : kConfigs) {
+    SCOPED_TRACE(::testing::Message() << "speed=" << test_cfg.speed
+                                      << " threads=" << test_cfg.threads);
+    aom_codec_enc_cfg_t cfg;
+    ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME),
+              AOM_CODEC_OK);
+    cfg.g_w = kWidth;
+    cfg.g_h = kHeight;
+    cfg.g_lag_in_frames = 0;
+    cfg.g_threads = test_cfg.threads;
+    cfg.rc_target_bitrate = 300;
+    cfg.rc_end_usage = AOM_CBR;
+
+    aom_codec_ctx_t enc;
+    ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK);
+    ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CPUUSED, test_cfg.speed),
+              AOM_CODEC_OK);
+    ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_ROW_MT, 1), AOM_CODEC_OK);
+    ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_ENABLE_CDEF, 1), AOM_CODEC_OK);
+
+    aom_svc_params_t svc_params = {};
+    svc_params.number_spatial_layers = 1;
+    svc_params.number_temporal_layers = 2;
+    svc_params.max_quantizers[0] = 56;
+    svc_params.min_quantizers[0] = 10;
+    svc_params.max_quantizers[1] = 56;
+    svc_params.min_quantizers[1] = 10;
+    svc_params.scaling_factor_num[0] = 1;
+    svc_params.scaling_factor_den[0] = 1;
+    svc_params.layer_target_bitrate[0] = cfg.rc_target_bitrate * 2 / 3;
+    svc_params.layer_target_bitrate[1] = cfg.rc_target_bitrate;
+    svc_params.framerate_factor[0] = 2;
+    svc_params.framerate_factor[1] = 1;
+    ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_PARAMS, &svc_params),
+              AOM_CODEC_OK);
+
+    aom_codec_dec_cfg_t dec_cfg = {};
+    dec_cfg.allow_lowbitdepth = 1;
+    aom_codec_ctx_t dec;
+    ASSERT_EQ(aom_codec_dec_init(&dec, aom_codec_av1_dx(), &dec_cfg, 0),
+              AOM_CODEC_OK);
+
+    aom_image_t *img =
+        aom_img_alloc(nullptr, AOM_IMG_FMT_I420, kWidth, kHeight, 1);
+    ASSERT_NE(img, nullptr);
+
+    for (int i = 0; i < kFrames; ++i) {
+      FillPannedScreen(img, i * 5);
+
+      aom_svc_layer_id_t layer_id = {};
+      layer_id.spatial_layer_id = 0;
+      layer_id.temporal_layer_id = i % 2;
+      ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_LAYER_ID, &layer_id),
+                AOM_CODEC_OK);
+
+      aom_svc_ref_frame_config_t ref_frame_config = {};
+      // Odd frames (TL1) do not refresh any reference slot -> non-reference.
+      ref_frame_config.refresh[0] = (i % 2 == 0) ? 1 : 0;
+      ref_frame_config.reference[0] = 1;
+      ref_frame_config.ref_idx[0] = 0;
+      ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SVC_REF_FRAME_CONFIG,
+                                  &ref_frame_config),
+                AOM_CODEC_OK);
+
+      ASSERT_EQ(aom_codec_encode(&enc, img, i, 1, AOM_EFLAG_CALCULATE_PSNR),
+                AOM_CODEC_OK);
+
+      const aom_image_t *decoded = nullptr;
+      bool had_psnr = false;
+      PsnrPkt psnr_pkt = {};
+      aom_codec_iter_t iter = nullptr;
+      while (const aom_codec_cx_pkt_t *pkt =
+                 aom_codec_get_cx_data(&enc, &iter)) {
+        if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) {
+          ASSERT_EQ(aom_codec_decode(
+                        &dec, static_cast<const uint8_t *>(pkt->data.frame.buf),
+                        pkt->data.frame.sz, nullptr),
+                    AOM_CODEC_OK);
+          aom_codec_iter_t dec_iter = nullptr;
+          decoded = aom_codec_get_frame(&dec, &dec_iter);
+        } else if (pkt->kind == AOM_CODEC_PSNR_PKT) {
+          had_psnr = true;
+          psnr_pkt = pkt->data.psnr;
+        }
+      }
+      ASSERT_NE(decoded, nullptr);
+      ASSERT_TRUE(had_psnr);
+
+      const PsnrPkt expected = ComputeI420Psnr(img, decoded);
+      for (int c = 0; c < 4; ++c) {
+        EXPECT_EQ(psnr_pkt.samples[c], expected.samples[c])
+            << "frame " << i << " component " << c;
+        EXPECT_EQ(psnr_pkt.sse[c], expected.sse[c])
+            << "frame " << i << " component " << c;
+        EXPECT_NEAR(psnr_pkt.psnr[c], expected.psnr[c], 1e-6)
+            << "frame " << i << " component " << c;
+      }
+    }
+
+    aom_img_free(img);
+    ASSERT_EQ(aom_codec_destroy(&dec), AOM_CODEC_OK);
+    ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+  }
+}
+#endif  // CONFIG_AV1_DECODER
+
 #if !CONFIG_REALTIME_ONLY
 TEST(EncodeAPI, PerFramePsnrGoodQualityZeroLagInFrames) {
   aom_codec_iface_t *iface = aom_codec_av1_cx();