Commit f9faa216c5 for aom
commit f9faa216c560b609f1b9a4711db62422000af7f8
Author: Lin Zheng <linzhen@google.com>
Date: Tue Sep 29 22:54:09 2026 +0000
Tweak AC coefficients in trellis optimization for noise patterns
Bug: 548701298
Change-Id: I44c268180d5a60a067cc1ed3c74264fbbdb25064
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index f9176d1676..169d181100 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -676,7 +676,7 @@ int av1_is_skip_txfm_penalized(const AV1_COMP *cpi, const MACROBLOCK *x,
if (src_var > rec_var) {
const int num_pixels = block_size_wide[bsize] * block_size_high[bsize];
const int64_t src_var_per_px = src_var / num_pixels;
- if (src_var_per_px >= 0) return 1;
+ if (src_var_per_px < 64) return 1;
}
}
#else
@@ -703,7 +703,7 @@ void av1_get_tx_skip_dist(const AV1_COMP *cpi, const MACROBLOCK *x,
const int num_pixels = block_size_wide[bsize] * block_size_high[bsize];
const int64_t src_var_per_px = src_var / num_pixels;
*no_skip_dist += var_offset;
- *skip_dist += (src_var_per_px >= 0) ? (var_offset * 4) : var_offset;
+ *skip_dist += (src_var_per_px < 64) ? (var_offset * 4) : var_offset;
}
}
#else
@@ -805,7 +805,7 @@ static void adjust_rdcost(const AV1_COMP *cpi, const MACROBLOCK *x,
const int64_t src_var_per_px = src_var / num_pixels;
if (var_offset > 0 &&
- ((is_skip_txfm && src_var_per_px >= 0) || is_smooth_intra_mode ||
+ ((is_skip_txfm && src_var_per_px < 64) || is_smooth_intra_mode ||
is_interintra_mode(mbmi) || has_second_ref(mbmi))) {
var_offset *= 4;
}
@@ -897,7 +897,7 @@ static void adjust_cost(const AV1_COMP *cpi, const MACROBLOCK *x,
const int64_t src_var_per_px = src_var / num_pixels;
if (var_offset > 0 &&
- ((is_skip_txfm && src_var_per_px >= 0) || is_smooth_intra_mode ||
+ ((is_skip_txfm && src_var_per_px < 64) || is_smooth_intra_mode ||
is_interintra_mode(mbmi) || has_second_ref(mbmi))) {
var_offset *= 4;
}
diff --git a/av1/encoder/txb_rdopt.c b/av1/encoder/txb_rdopt.c
index e23630a880..aaa74984b3 100644
--- a/av1/encoder/txb_rdopt.c
+++ b/av1/encoder/txb_rdopt.c
@@ -11,6 +11,7 @@
#include "av1/encoder/txb_rdopt.h"
#include "av1/encoder/txb_rdopt_utils.h"
+#include "av1/encoder/rdopt_utils.h"
#include "aom_ports/mem.h"
#include "av1/common/idct.h"
@@ -190,7 +191,7 @@ static AOM_FORCE_INLINE void update_coeff_eob(
const int16_t *scan, const LV_MAP_EOB_COST *txb_eob_costs,
const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff,
tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness,
- const qm_val_t *iqmatrix, const qm_val_t *qmatrix) {
+ const qm_val_t *iqmatrix, const qm_val_t *qmatrix, int is_noise_pattern) {
assert(si != *eob - 1);
const int ci = scan[si];
const tran_low_t qc = qcoeff[ci];
@@ -272,7 +273,8 @@ static AOM_FORCE_INLINE void update_coeff_eob(
}
}
- const int qc_threshold = (si <= 5) ? 2 : 1;
+ const int min_eob_cutoff = is_noise_pattern ? 8 : 5;
+ const int qc_threshold = (si <= min_eob_cutoff) ? 2 : 1;
const int allow_lower_qc = sharpness ? abs_qc > qc_threshold : 1;
if (allow_lower_qc) {
@@ -284,7 +286,7 @@ static AOM_FORCE_INLINE void update_coeff_eob(
}
}
- if ((sharpness == 0 || new_eob >= 5) && rd_new_eob < rd) {
+ if ((sharpness == 0 || new_eob >= min_eob_cutoff) && rd_new_eob < rd) {
for (int ni = 0; ni < *nz_num; ++ni) {
int last_ci = nz_ci[ni];
levels[get_padded_idx(last_ci, bhl)] = 0;
@@ -374,12 +376,13 @@ static AOM_FORCE_INLINE void update_coeff_eob_facade(
const int16_t *scan, const LV_MAP_EOB_COST *txb_eob_costs,
const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff,
tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness,
- const qm_val_t *iqmatrix, const qm_val_t *qmatrix, int max_nz_num) {
+ const qm_val_t *iqmatrix, const qm_val_t *qmatrix, int max_nz_num,
+ int is_noise_pattern) {
for (; *si >= 0 && *nz_num <= max_nz_num; --*si) {
update_coeff_eob(accu_rate, accu_dist, eob, nz_num, nz_ci, *si, tx_size,
tx_class, bhl, width, dc_sign_ctx, rdmult, shift, dequant,
scan, txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff,
- levels, sharpness, iqmatrix, qmatrix);
+ levels, sharpness, iqmatrix, qmatrix, is_noise_pattern);
}
}
@@ -440,6 +443,22 @@ int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
const LV_MAP_EOB_COST *txb_eob_costs =
&coeff_costs->eob_costs[eob_multi_size][plane_type];
+ int is_noise_pattern = 0;
+#if CONFIG_AV1_HIGHBITDEPTH
+ if (xd->bd > 8 && sharpness == 3 && plane == AOM_PLANE_Y) {
+ int64_t src_var, rec_var;
+ av1_get_variance_stats(x, &src_var, &rec_var);
+ if (src_var > rec_var) {
+ const int num_pixels =
+ block_size_wide[mbmi->bsize] * block_size_high[mbmi->bsize];
+ const int64_t src_var_per_px = src_var / num_pixels;
+ if (src_var_per_px < 64) {
+ is_noise_pattern = 1;
+ }
+ }
+ }
+#endif
+
// For the IQ and SSIMULACRA 2 tunings, increase rshift from 2 to 4.
// This biases trellis quantization towards keeping more coefficients, and
// together with the IQ and SSIMULACRA2 rdmult adjustment in
@@ -449,7 +468,7 @@ int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
const int rshift = (cpi->oxcf.tune_cfg.tuning == AOM_TUNE_IQ ||
cpi->oxcf.tune_cfg.tuning == AOM_TUNE_SSIMULACRA2)
? 7
- : 5;
+ : (is_noise_pattern ? 6 : 5);
const int(*trellis_rd_mult)[2] = cpi->sf.tx_sf.use_chroma_trellis_rd_mult
? plane_rd_mult_chroma
@@ -499,13 +518,13 @@ int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
--si;
}
-#define UPDATE_COEFF_EOB_CASE(tx_class_literal) \
- case tx_class_literal: \
- update_coeff_eob_facade( \
- &accu_rate, &accu_dist, &eob, &nz_num, nz_ci, &si, tx_size, \
- tx_class_literal, bhl, width, txb_ctx->dc_sign_ctx, rdmult, shift, \
- dequant, scan, txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff, \
- levels, sharpness, iqmatrix, qmatrix, max_nz_num); \
+#define UPDATE_COEFF_EOB_CASE(tx_class_literal) \
+ case tx_class_literal: \
+ update_coeff_eob_facade( \
+ &accu_rate, &accu_dist, &eob, &nz_num, nz_ci, &si, tx_size, \
+ tx_class_literal, bhl, width, txb_ctx->dc_sign_ctx, rdmult, shift, \
+ dequant, scan, txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff, \
+ levels, sharpness, iqmatrix, qmatrix, max_nz_num, is_noise_pattern); \
break
switch (tx_class) {
UPDATE_COEFF_EOB_CASE(TX_CLASS_2D);