Commit 7a210280be for aom
commit 7a210280be0986d63abae0661b71d0c342560b9d
Author: Jingning Han <jingning@google.com>
Date: Thu Aug 13 11:37:51 2026 -0700
Properly scale the distortion in high bit-depth settings
Scale the high bit-depth distortion to align with the baseline
rate-distortion optimization usage.
BUG=b/435430898
Change-Id: I7c5dc2636086cb4c9001786f77e35f82ace77c58
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 5f0edb1872..b0ea57449a 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -632,11 +632,15 @@ static void get_variance_stats_hbd(const MACROBLOCK *x, int64_t *src_var,
BLOCK_SIZE bsize = mbmi->bsize;
int bw = block_size_wide[bsize];
int bh = block_size_high[bsize];
+ const int shift = 2 * (xd->bd - 8);
*rec_var = aom_highbd_calc_variance_stat(CONVERT_TO_SHORTPTR(pd->dst.buf),
pd->dst.stride, bw, bh);
*src_var = aom_highbd_calc_variance_stat(CONVERT_TO_SHORTPTR(p->src.buf),
p->src.stride, bw, bh);
+
+ *rec_var = ROUND_POWER_OF_TWO(*rec_var, shift);
+ *src_var = ROUND_POWER_OF_TWO(*src_var, shift);
}
#endif // CONFIG_AV1_HIGHBITDEPTH