Commit 808aa15eab for aom
commit 808aa15eabe43d4847da524e5e1d8abde5490143
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Oct 31 09:54:06 2026 -0700
av1: Inline txfm/quant wrappers and eob==1 path
Inline hot transform, quantize, and block subtraction wrappers in
encodemb.h and idct.h, add a DC-only (eob == 1) fast path in
av1_optimize_txb that skips 2D levels buffer initialization, and split
non-DCT 8x8, 16x16, 8x16, and 16x8 forward transforms into out-of-line
helpers.
Benchmarking results (incremental vs prev CL / cumulative vs baseline):
- Stock avifenc still-image (Default 1T): -2.71% / -30.75% encode time
- Stock avifenc still-image (Default 8T): -0.95% / -23.92% encode time
- 48-image multi-rate corpus (Full Suite 1T): -2.66% / -31.11% time
- Bit-exact output (1T and 8T)
TAG=agy
CONV=b929a630-a6cd-45a0-979c-1767c30178ee
Change-Id: Id522b23fee242deac3c9e1d6c19daf1649e38a47
diff --git a/av1/common/idct.c b/av1/common/idct.c
index 7c8e3d6e5a..6f914faf86 100644
--- a/av1/common/idct.c
+++ b/av1/common/idct.c
@@ -204,20 +204,6 @@ static void highbd_inv_txfm_add_64x64_c(const tran_low_t *input, uint8_t *dest,
bd);
}
-static void init_txfm_param(const MACROBLOCKD *xd, int plane, TX_SIZE tx_size,
- TX_TYPE tx_type, int eob, int reduced_tx_set,
- TxfmParam *txfm_param) {
- (void)plane;
- txfm_param->tx_type = tx_type;
- txfm_param->tx_size = tx_size;
- txfm_param->eob = eob;
- txfm_param->lossless = xd->lossless[xd->mi[0]->segment_id];
- txfm_param->bd = xd->bd;
- txfm_param->is_hbd = is_cur_buf_hbd(xd);
- txfm_param->tx_set_type = av1_get_ext_tx_set_type(
- txfm_param->tx_size, is_inter_block(xd->mi[0]), reduced_tx_set);
-}
-
void av1_highbd_inv_txfm_add_c(const tran_low_t *input, uint8_t *dest,
int stride, const TxfmParam *txfm_param) {
assert(av1_ext_tx_used[txfm_param->tx_set_type][txfm_param->tx_type]);
@@ -309,23 +295,3 @@ void av1_inv_txfm_add_c(const tran_low_t *dqcoeff, uint8_t *dst, int stride,
}
}
}
-
-void av1_inverse_transform_block(const MACROBLOCKD *xd,
- const tran_low_t *dqcoeff, int plane,
- TX_TYPE tx_type, TX_SIZE tx_size, uint8_t *dst,
- int stride, int eob, int reduced_tx_set) {
- if (!eob) return;
-
- assert(eob <= av1_get_max_eob(tx_size));
-
- TxfmParam txfm_param;
- init_txfm_param(xd, plane, tx_size, tx_type, eob, reduced_tx_set,
- &txfm_param);
- assert(av1_ext_tx_used[txfm_param.tx_set_type][txfm_param.tx_type]);
-
- if (txfm_param.is_hbd) {
- av1_highbd_inv_txfm_add(dqcoeff, dst, stride, &txfm_param);
- } else {
- av1_inv_txfm_add(dqcoeff, dst, stride, &txfm_param);
- }
-}
diff --git a/av1/common/idct.h b/av1/common/idct.h
index 3f4c98af6e..2b98874d31 100644
--- a/av1/common/idct.h
+++ b/av1/common/idct.h
@@ -15,6 +15,7 @@
#include <assert.h>
#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
#include "av1/common/blockd.h"
#include "av1/common/common.h"
@@ -40,10 +41,38 @@ static inline int av1_get_tx_scale(const TX_SIZE tx_size) {
return tx_scale[tx_size];
}
-void av1_inverse_transform_block(const MACROBLOCKD *xd,
- const tran_low_t *dqcoeff, int plane,
- TX_TYPE tx_type, TX_SIZE tx_size, uint8_t *dst,
- int stride, int eob, int reduced_tx_set);
+static inline void av1_inverse_transform_block(const MACROBLOCKD *xd,
+ const tran_low_t *dqcoeff,
+ int plane, TX_TYPE tx_type,
+ TX_SIZE tx_size, uint8_t *dst,
+ int stride, int eob,
+ int reduced_tx_set) {
+ if (!eob) return;
+
+ assert(eob <= av1_get_max_eob(tx_size));
+ (void)plane;
+
+ TxfmParam txfm_param;
+ txfm_param.tx_type = tx_type;
+ txfm_param.tx_size = tx_size;
+ txfm_param.eob = eob;
+ txfm_param.lossless = xd->lossless[xd->mi[0]->segment_id];
+ txfm_param.bd = xd->bd;
+ txfm_param.is_hbd = is_cur_buf_hbd(xd);
+#if !defined(NDEBUG)
+ txfm_param.tx_set_type = av1_get_ext_tx_set_type(
+ txfm_param.tx_size, is_inter_block(xd->mi[0]), reduced_tx_set);
+ assert(av1_ext_tx_used[txfm_param.tx_set_type][txfm_param.tx_type]);
+#else
+ (void)reduced_tx_set;
+#endif
+
+ if (UNLIKELY(txfm_param.is_hbd)) {
+ av1_highbd_inv_txfm_add(dqcoeff, dst, stride, &txfm_param);
+ } else {
+ av1_inv_txfm_add(dqcoeff, dst, stride, &txfm_param);
+ }
+}
void av1_highbd_iwht4x4_add(const tran_low_t *input, uint8_t *dest, int stride,
int eob, int bd);
diff --git a/av1/common/txb_common.h b/av1/common/txb_common.h
index b57bd5da22..7c1c68531b 100644
--- a/av1/common/txb_common.h
+++ b/av1/common/txb_common.h
@@ -414,6 +414,33 @@ static inline void set_dc_sign(int *cul_level, int dc_val) {
*cul_level += ((dc_val < 0) + ((dc_val > 0) << 1)) << COEFF_CONTEXT_BITS;
}
+static inline uint8_t av1_get_txb_entropy_context(const tran_low_t *qcoeff,
+ const SCAN_ORDER *scan_order,
+ int eob) {
+ if (eob == 0) return 0;
+ const int32_t dc_val = qcoeff[0];
+ int cul_level = abs(dc_val);
+ if (cul_level < COEFF_CONTEXT_MASK && eob > 1) {
+ const int16_t *const scan = scan_order->scan;
+ int c = 1;
+ for (; c + 3 < eob; c += 4) {
+ cul_level += abs(qcoeff[scan[c]]) + abs(qcoeff[scan[c + 1]]) +
+ abs(qcoeff[scan[c + 2]]) + abs(qcoeff[scan[c + 3]]);
+ if (cul_level >= COEFF_CONTEXT_MASK) break;
+ }
+ if (cul_level < COEFF_CONTEXT_MASK) {
+ for (; c < eob; ++c) {
+ cul_level += abs(qcoeff[scan[c]]);
+ }
+ }
+ }
+
+ cul_level = AOMMIN(COEFF_CONTEXT_MASK, cul_level);
+ set_dc_sign(&cul_level, dc_val);
+
+ return (uint8_t)cul_level;
+}
+
#define MAX_TX_SIZE_UNIT 16
static const int8_t kTxbCtxSigns[3] = { 0, -1, 1 };
static const int8_t kTxbCtxDcSignContexts[4 * MAX_TX_SIZE_UNIT + 1] = {
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index d64390c417..73d1a7b628 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -140,30 +140,10 @@ static inline void fill_residue_outside_frame(
}
}
-void av1_subtract_block(const MACROBLOCK *x, int rows, int cols, int16_t *diff,
- ptrdiff_t diff_stride, const uint8_t *src8,
- ptrdiff_t src_stride, const uint8_t *pred8,
- ptrdiff_t pred_stride, int plane,
- BLOCK_SIZE plane_bsize, int blk_col, int blk_row,
- TX_TYPE tx_type, bool do_border_pad) {
- assert(rows >= 4 && cols >= 4);
- const MACROBLOCKD *const xd = &x->e_mbd;
- BitDepthInfo bd_info = get_bit_depth_info(xd);
-#if CONFIG_AV1_HIGHBITDEPTH
- if (bd_info.use_highbitdepth_buf) {
- aom_highbd_subtract_block(rows, cols, diff, diff_stride, src8, src_stride,
- pred8, pred_stride);
- } else {
- aom_subtract_block(rows, cols, diff, diff_stride, src8, src_stride, pred8,
- pred_stride);
- }
-#else
- (void)bd_info;
- aom_subtract_block(rows, cols, diff, diff_stride, src8, src_stride, pred8,
- pred_stride);
-#endif
- if (!do_border_pad) return;
-
+void av1_pad_block_border(const MACROBLOCK *x, int rows, int cols,
+ int16_t *diff, ptrdiff_t diff_stride, int plane,
+ BLOCK_SIZE plane_bsize, int blk_col, int blk_row,
+ TX_TYPE tx_type) {
int visible_cols, visible_rows;
const int is_border_block = get_visible_dimensions(
x, plane, plane_bsize, blk_col, blk_row, cols, rows,
@@ -173,56 +153,6 @@ void av1_subtract_block(const MACROBLOCK *x, int rows, int cols, int16_t *diff,
visible_rows, tx_type);
}
-void av1_subtract_txb(MACROBLOCK *x, int plane, BLOCK_SIZE plane_bsize,
- int blk_col, int blk_row, TX_SIZE tx_size,
- TX_TYPE tx_type, bool do_border_pad) {
- struct macroblock_plane *const p = &x->plane[plane];
- const struct macroblockd_plane *const pd = &x->e_mbd.plane[plane];
- const int diff_stride = block_size_wide[plane_bsize];
- const int src_stride = p->src.stride;
- const int dst_stride = pd->dst.stride;
- const int tx1d_width = tx_size_wide[tx_size];
- const int tx1d_height = tx_size_high[tx_size];
- uint8_t *dst = &pd->dst.buf[(blk_row * dst_stride + blk_col) << MI_SIZE_LOG2];
- uint8_t *src = &p->src.buf[(blk_row * src_stride + blk_col) << MI_SIZE_LOG2];
- int16_t *src_diff =
- &p->src_diff[(blk_row * diff_stride + blk_col) << MI_SIZE_LOG2];
- av1_subtract_block(x, tx1d_height, tx1d_width, src_diff, diff_stride, src,
- src_stride, dst, dst_stride, plane, plane_bsize, blk_col,
- blk_row, tx_type, do_border_pad);
-}
-
-void av1_subtract_plane(MACROBLOCK *x, BLOCK_SIZE plane_bsize, int plane,
- bool do_border_pad) {
- struct macroblock_plane *const p = &x->plane[plane];
- const struct macroblockd_plane *const pd = &x->e_mbd.plane[plane];
- assert(plane_bsize < BLOCK_SIZES_ALL);
- const int bw = block_size_wide[plane_bsize];
- const int bh = block_size_high[plane_bsize];
-
- av1_subtract_block(x, bh, bw, p->src_diff, bw, p->src.buf, p->src.stride,
- pd->dst.buf, pd->dst.stride, plane, plane_bsize, 0, 0,
- DCT_DCT, do_border_pad);
-}
-
-int av1_optimize_b(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
- int block, TX_SIZE tx_size, TX_TYPE tx_type,
- const TXB_CTX *const txb_ctx, int *rate_cost) {
- MACROBLOCKD *const xd = &x->e_mbd;
- struct macroblock_plane *const p = &x->plane[plane];
- const int eob = p->eobs[block];
- const int segment_id = xd->mi[0]->segment_id;
-
- if (eob == 0 || !cpi->optimize_seg_arr[segment_id] ||
- xd->lossless[segment_id]) {
- *rate_cost = av1_cost_skip_txb(&x->coeff_costs, txb_ctx, plane, tx_size);
- return eob;
- }
-
- return av1_optimize_txb(cpi, x, plane, block, tx_size, tx_type, txb_ctx,
- rate_cost, cpi->oxcf.algo_cfg.sharpness);
-}
-
// Hyper-parameters for dropout optimization, based on following logics.
// TODO(yjshen): These settings are tuned by experiments. They may still be
// optimized for better performance.
@@ -369,26 +299,6 @@ void av1_dropout_qcoeff_num(MACROBLOCK *mb, int plane, int block,
}
}
-enum {
- QUANT_FUNC_LOWBD = 0,
- QUANT_FUNC_HIGHBD = 1,
- QUANT_FUNC_TYPES = 2
-} UENUM1BYTE(QUANT_FUNC);
-
-#if CONFIG_AV1_HIGHBITDEPTH
-static const AV1_QUANT_FACADE
- quant_func_list[AV1_XFORM_QUANT_TYPES][QUANT_FUNC_TYPES] = {
- { av1_quantize_fp_facade, av1_highbd_quantize_fp_facade },
- { av1_quantize_b_facade, av1_highbd_quantize_b_facade },
- { av1_quantize_dc_facade, av1_highbd_quantize_dc_facade },
- { NULL, NULL }
- };
-#else
-static const AV1_QUANT_FACADE quant_func_list[AV1_XFORM_QUANT_TYPES] = {
- av1_quantize_fp_facade, av1_quantize_b_facade, av1_quantize_dc_facade, NULL
-};
-#endif
-
// Computes the transform for DC only blocks
void av1_xform_dc_only(MACROBLOCK *x, int plane, int block,
TxfmParam *txfm_param, int64_t per_px_mean) {
@@ -402,104 +312,6 @@ void av1_xform_dc_only(MACROBLOCK *x, int plane, int block,
(tran_low_t)((per_px_mean * dc_coeff_scale[txfm_param->tx_size]) >> 12);
}
-void av1_xform_quant(MACROBLOCK *x, int plane, int block, int blk_row,
- int blk_col, BLOCK_SIZE plane_bsize, TxfmParam *txfm_param,
- const QUANT_PARAM *qparam) {
- av1_xform(x, plane, block, blk_row, blk_col, plane_bsize, txfm_param);
- av1_quant(x, plane, block, txfm_param, qparam);
-}
-
-void av1_xform(MACROBLOCK *x, int plane, int block, int blk_row, int blk_col,
- BLOCK_SIZE plane_bsize, TxfmParam *txfm_param) {
- const struct macroblock_plane *const p = &x->plane[plane];
- const int block_offset = BLOCK_OFFSET(block);
- tran_low_t *const coeff = p->coeff + block_offset;
- const int diff_stride = block_size_wide[plane_bsize];
-
- const int src_offset = (blk_row * diff_stride + blk_col);
- const int16_t *src_diff = &p->src_diff[src_offset << MI_SIZE_LOG2];
-
- av1_fwd_txfm(src_diff, coeff, diff_stride, txfm_param);
-}
-
-void av1_quant(MACROBLOCK *x, int plane, int block, TxfmParam *txfm_param,
- const QUANT_PARAM *qparam) {
- const struct macroblock_plane *const p = &x->plane[plane];
- const SCAN_ORDER *const scan_order =
- get_scan(txfm_param->tx_size, txfm_param->tx_type);
- const int block_offset = BLOCK_OFFSET(block);
- tran_low_t *const coeff = p->coeff + block_offset;
- tran_low_t *const qcoeff = p->qcoeff + block_offset;
- tran_low_t *const dqcoeff = p->dqcoeff + block_offset;
- uint16_t *const eob = &p->eobs[block];
-
- if (qparam->xform_quant_idx != AV1_XFORM_QUANT_SKIP_QUANT) {
- const int n_coeffs = av1_get_max_eob(txfm_param->tx_size);
- if (LIKELY(!x->seg_skip_block)) {
-#if CONFIG_AV1_HIGHBITDEPTH
- if (LIKELY(!txfm_param->is_hbd &&
- qparam->xform_quant_idx == AV1_XFORM_QUANT_FP)) {
- av1_quantize_fp_facade(coeff, n_coeffs, p, qcoeff, dqcoeff, eob,
- scan_order, qparam);
- } else {
- quant_func_list[qparam->xform_quant_idx][txfm_param->is_hbd](
- coeff, n_coeffs, p, qcoeff, dqcoeff, eob, scan_order, qparam);
- }
-#else
- if (LIKELY(qparam->xform_quant_idx == AV1_XFORM_QUANT_FP)) {
- av1_quantize_fp_facade(coeff, n_coeffs, p, qcoeff, dqcoeff, eob,
- scan_order, qparam);
- } else {
- quant_func_list[qparam->xform_quant_idx](
- coeff, n_coeffs, p, qcoeff, dqcoeff, eob, scan_order, qparam);
- }
-#endif
- } else {
- av1_quantize_skip(n_coeffs, qcoeff, dqcoeff, eob);
- }
- }
- // use_optimize_b is true means av1_optimze_b will be called,
- // thus cannot update entropy ctx now (performed in optimize_b)
- if (qparam->use_optimize_b) {
- p->txb_entropy_ctx[block] = 0;
- } else {
- p->txb_entropy_ctx[block] =
- av1_get_txb_entropy_context(qcoeff, scan_order, *eob);
- }
-}
-
-void av1_setup_xform(const AV1_COMMON *cm, MACROBLOCK *x, TX_SIZE tx_size,
- TX_TYPE tx_type, TxfmParam *txfm_param) {
- MACROBLOCKD *const xd = &x->e_mbd;
- MB_MODE_INFO *const mbmi = xd->mi[0];
-
- txfm_param->tx_type = tx_type;
- txfm_param->tx_size = tx_size;
- txfm_param->lossless = xd->lossless[mbmi->segment_id];
- txfm_param->tx_set_type = av1_get_ext_tx_set_type(
- tx_size, is_inter_block(mbmi), cm->features.reduced_tx_set_used);
-
- txfm_param->bd = xd->bd;
- txfm_param->is_hbd = is_cur_buf_hbd(xd);
-}
-void av1_setup_quant(TX_SIZE tx_size, int use_optimize_b, int xform_quant_idx,
- int use_quant_b_adapt, QUANT_PARAM *qparam) {
- qparam->log_scale = av1_get_tx_scale(tx_size);
- qparam->tx_size = tx_size;
-
- qparam->use_quant_b_adapt = use_quant_b_adapt;
-
- // TODO(bohanli): optimize_b and quantization idx has relationship,
- // but is kind of buried and complicated in different encoding stages.
- // Should have a unified function to derive quant_idx, rather than
- // determine and pass in the quant_idx
- qparam->use_optimize_b = use_optimize_b;
- qparam->xform_quant_idx = xform_quant_idx;
-
- qparam->qmatrix = NULL;
- qparam->iqmatrix = NULL;
-}
-
static void encode_block(int plane, int block, int blk_row, int blk_col,
BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg,
RUN_TYPE dry_run) {
diff --git a/av1/encoder/encodemb.h b/av1/encoder/encodemb.h
index 3f8a5a5d05..a6331f0f16 100644
--- a/av1/encoder/encodemb.h
+++ b/av1/encoder/encodemb.h
@@ -13,11 +13,15 @@
#define AOM_AV1_ENCODER_ENCODEMB_H_
#include "config/aom_config.h"
+#include "config/aom_dsp_rtcd.h"
#include "av1/common/av1_common_int.h"
+#include "av1/common/idct.h"
+#include "av1/common/scan.h"
#include "av1/common/txb_common.h"
#include "av1/encoder/av1_quantize.h"
#include "av1/encoder/block.h"
+#include "av1/encoder/hybrid_fwd_txfm.h"
#include "av1/encoder/tokenize.h"
#ifdef __cplusplus
extern "C" {
@@ -72,10 +76,40 @@ void av1_foreach_transformed_block_in_plane(
void av1_encode_sby_pass1(struct AV1_COMP *cpi, MACROBLOCK *x,
BLOCK_SIZE bsize);
-void av1_setup_xform(const AV1_COMMON *cm, MACROBLOCK *x, TX_SIZE tx_size,
- TX_TYPE tx_type, TxfmParam *txfm_param);
-void av1_setup_quant(TX_SIZE tx_size, int use_optimize_b, int xform_quant_idx,
- int use_quant_b_adapt, QUANT_PARAM *qparam);
+static inline void av1_setup_xform(const AV1_COMMON *cm, MACROBLOCK *x,
+ TX_SIZE tx_size, TX_TYPE tx_type,
+ TxfmParam *txfm_param) {
+ MACROBLOCKD *const xd = &x->e_mbd;
+ MB_MODE_INFO *const mbmi = xd->mi[0];
+
+ txfm_param->tx_type = tx_type;
+ txfm_param->tx_size = tx_size;
+ txfm_param->lossless = xd->lossless[mbmi->segment_id];
+#if !defined(NDEBUG)
+ txfm_param->tx_set_type = av1_get_ext_tx_set_type(
+ tx_size, is_inter_block(mbmi), cm->features.reduced_tx_set_used);
+#else
+ (void)cm;
+#endif
+
+ txfm_param->bd = xd->bd;
+ txfm_param->is_hbd = is_cur_buf_hbd(xd);
+}
+
+static inline void av1_setup_quant(TX_SIZE tx_size, int use_optimize_b,
+ int xform_quant_idx, int use_quant_b_adapt,
+ QUANT_PARAM *qparam) {
+ qparam->log_scale = av1_get_tx_scale(tx_size);
+ qparam->tx_size = tx_size;
+
+ qparam->use_quant_b_adapt = use_quant_b_adapt;
+ qparam->use_optimize_b = use_optimize_b;
+ qparam->xform_quant_idx = xform_quant_idx;
+
+ qparam->qmatrix = NULL;
+ qparam->iqmatrix = NULL;
+}
+
static inline void av1_setup_qmatrix(const CommonQuantParams *quant_params,
const MACROBLOCKD *xd, int plane,
TX_SIZE tx_size, TX_TYPE tx_type,
@@ -95,19 +129,90 @@ static inline void av1_setup_qmatrix(const CommonQuantParams *quant_params,
void av1_xform_dc_only(MACROBLOCK *x, int plane, int block,
TxfmParam *txfm_param, int64_t per_px_mean);
-void av1_xform_quant(MACROBLOCK *x, int plane, int block, int blk_row,
- int blk_col, BLOCK_SIZE plane_bsize, TxfmParam *txfm_param,
- const QUANT_PARAM *qparam);
+static inline void av1_xform(MACROBLOCK *x, int plane, int block, int blk_row,
+ int blk_col, BLOCK_SIZE plane_bsize,
+ TxfmParam *txfm_param) {
+ const struct macroblock_plane *const p = &x->plane[plane];
+ const int block_offset = BLOCK_OFFSET(block);
+ tran_low_t *const coeff = p->coeff + block_offset;
+ const int diff_stride = block_size_wide[plane_bsize];
-void av1_xform(MACROBLOCK *x, int plane, int block, int blk_row, int blk_col,
- BLOCK_SIZE plane_bsize, TxfmParam *txfm_param);
+ const int src_offset = (blk_row * diff_stride + blk_col);
+ const int16_t *src_diff = &p->src_diff[src_offset << MI_SIZE_LOG2];
-void av1_quant(MACROBLOCK *x, int plane, int block, TxfmParam *txfm_param,
- const QUANT_PARAM *qparam);
+ av1_fwd_txfm(src_diff, coeff, diff_stride, txfm_param);
+}
-int av1_optimize_b(const struct AV1_COMP *cpi, MACROBLOCK *mb, int plane,
- int block, TX_SIZE tx_size, TX_TYPE tx_type,
- const TXB_CTX *const txb_ctx, int *rate_cost);
+static inline void av1_quant(MACROBLOCK *x, int plane, int block,
+ TxfmParam *txfm_param, const QUANT_PARAM *qparam) {
+ const struct macroblock_plane *const p = &x->plane[plane];
+ const SCAN_ORDER *const scan_order =
+ get_scan(txfm_param->tx_size, txfm_param->tx_type);
+ const int block_offset = BLOCK_OFFSET(block);
+ tran_low_t *const coeff = p->coeff + block_offset;
+ tran_low_t *const qcoeff = p->qcoeff + block_offset;
+ tran_low_t *const dqcoeff = p->dqcoeff + block_offset;
+ uint16_t *const eob = &p->eobs[block];
+
+ if (LIKELY(qparam->xform_quant_idx != AV1_XFORM_QUANT_SKIP_QUANT)) {
+ const int n_coeffs = av1_get_max_eob(txfm_param->tx_size);
+ if (LIKELY(!x->seg_skip_block)) {
+#if CONFIG_AV1_HIGHBITDEPTH
+ if (LIKELY(!txfm_param->is_hbd)) {
+ if (LIKELY(qparam->xform_quant_idx == AV1_XFORM_QUANT_FP)) {
+ av1_quantize_fp_facade(coeff, n_coeffs, p, qcoeff, dqcoeff, eob,
+ scan_order, qparam);
+ } else if (qparam->xform_quant_idx == AV1_XFORM_QUANT_B) {
+ av1_quantize_b_facade(coeff, n_coeffs, p, qcoeff, dqcoeff, eob,
+ scan_order, qparam);
+ } else {
+ av1_quantize_dc_facade(coeff, n_coeffs, p, qcoeff, dqcoeff, eob,
+ scan_order, qparam);
+ }
+ } else {
+ if (qparam->xform_quant_idx == AV1_XFORM_QUANT_FP) {
+ av1_highbd_quantize_fp_facade(coeff, n_coeffs, p, qcoeff, dqcoeff,
+ eob, scan_order, qparam);
+ } else if (qparam->xform_quant_idx == AV1_XFORM_QUANT_B) {
+ av1_highbd_quantize_b_facade(coeff, n_coeffs, p, qcoeff, dqcoeff, eob,
+ scan_order, qparam);
+ } else {
+ av1_highbd_quantize_dc_facade(coeff, n_coeffs, p, qcoeff, dqcoeff,
+ eob, scan_order, qparam);
+ }
+ }
+#else
+ if (LIKELY(qparam->xform_quant_idx == AV1_XFORM_QUANT_FP)) {
+ av1_quantize_fp_facade(coeff, n_coeffs, p, qcoeff, dqcoeff, eob,
+ scan_order, qparam);
+ } else if (qparam->xform_quant_idx == AV1_XFORM_QUANT_B) {
+ av1_quantize_b_facade(coeff, n_coeffs, p, qcoeff, dqcoeff, eob,
+ scan_order, qparam);
+ } else {
+ av1_quantize_dc_facade(coeff, n_coeffs, p, qcoeff, dqcoeff, eob,
+ scan_order, qparam);
+ }
+#endif
+ } else {
+ av1_quantize_skip(n_coeffs, qcoeff, dqcoeff, eob);
+ }
+ }
+ if (qparam->use_optimize_b) {
+ p->txb_entropy_ctx[block] = 0;
+ } else {
+ p->txb_entropy_ctx[block] =
+ av1_get_txb_entropy_context(qcoeff, scan_order, *eob);
+ }
+}
+
+static inline void av1_xform_quant(MACROBLOCK *x, int plane, int block,
+ int blk_row, int blk_col,
+ BLOCK_SIZE plane_bsize,
+ TxfmParam *txfm_param,
+ const QUANT_PARAM *qparam) {
+ av1_xform(x, plane, block, blk_row, blk_col, plane_bsize, txfm_param);
+ av1_quant(x, plane, block, txfm_param, qparam);
+}
// This function can be used as (i) a further optimization to reduce the
// redundancy of quantized coefficients (a.k.a., `qcoeff`) after trellis
@@ -140,19 +245,70 @@ void av1_dropout_qcoeff_num(MACROBLOCK *mb, int plane, int block,
TX_SIZE tx_size, TX_TYPE tx_type,
int dropout_num_before, int dropout_num_after);
-void av1_subtract_block(const MACROBLOCK *x, int rows, int cols, int16_t *diff,
- ptrdiff_t diff_stride, const uint8_t *src8,
- ptrdiff_t src_stride, const uint8_t *pred8,
- ptrdiff_t pred_stride, int plane,
- BLOCK_SIZE plane_bsize, int blk_col, int blk_row,
- TX_TYPE tx_type, bool do_border_pad);
+void av1_pad_block_border(const MACROBLOCK *x, int rows, int cols,
+ int16_t *diff, ptrdiff_t diff_stride, int plane,
+ BLOCK_SIZE plane_bsize, int blk_col, int blk_row,
+ TX_TYPE tx_type);
-void av1_subtract_txb(MACROBLOCK *x, int plane, BLOCK_SIZE plane_bsize,
- int blk_col, int blk_row, TX_SIZE tx_size,
- TX_TYPE tx_type, bool do_border_pad);
+static inline void av1_subtract_block(const MACROBLOCK *x, int rows, int cols,
+ int16_t *diff, ptrdiff_t diff_stride,
+ const uint8_t *src8, ptrdiff_t src_stride,
+ const uint8_t *pred8,
+ ptrdiff_t pred_stride, int plane,
+ BLOCK_SIZE plane_bsize, int blk_col,
+ int blk_row, TX_TYPE tx_type,
+ bool do_border_pad) {
+ assert(rows >= 4 && cols >= 4);
+#if CONFIG_AV1_HIGHBITDEPTH
+ if (UNLIKELY(is_cur_buf_hbd(&x->e_mbd))) {
+ aom_highbd_subtract_block(rows, cols, diff, diff_stride, src8, src_stride,
+ pred8, pred_stride);
+ } else {
+ aom_subtract_block(rows, cols, diff, diff_stride, src8, src_stride, pred8,
+ pred_stride);
+ }
+#else
+ aom_subtract_block(rows, cols, diff, diff_stride, src8, src_stride, pred8,
+ pred_stride);
+#endif
+ if (UNLIKELY(do_border_pad)) {
+ av1_pad_block_border(x, rows, cols, diff, diff_stride, plane, plane_bsize,
+ blk_col, blk_row, tx_type);
+ }
+}
+
+static inline void av1_subtract_txb(MACROBLOCK *x, int plane,
+ BLOCK_SIZE plane_bsize, int blk_col,
+ int blk_row, TX_SIZE tx_size,
+ TX_TYPE tx_type, bool do_border_pad) {
+ struct macroblock_plane *const p = &x->plane[plane];
+ const struct macroblockd_plane *const pd = &x->e_mbd.plane[plane];
+ const int diff_stride = block_size_wide[plane_bsize];
+ const int src_stride = p->src.stride;
+ const int dst_stride = pd->dst.stride;
+ const int tx1d_width = tx_size_wide[tx_size];
+ const int tx1d_height = tx_size_high[tx_size];
+ uint8_t *dst = &pd->dst.buf[(blk_row * dst_stride + blk_col) << MI_SIZE_LOG2];
+ uint8_t *src = &p->src.buf[(blk_row * src_stride + blk_col) << MI_SIZE_LOG2];
+ int16_t *src_diff =
+ &p->src_diff[(blk_row * diff_stride + blk_col) << MI_SIZE_LOG2];
+ av1_subtract_block(x, tx1d_height, tx1d_width, src_diff, diff_stride, src,
+ src_stride, dst, dst_stride, plane, plane_bsize, blk_col,
+ blk_row, tx_type, do_border_pad);
+}
+
+static inline void av1_subtract_plane(MACROBLOCK *x, BLOCK_SIZE plane_bsize,
+ int plane, bool do_border_pad) {
+ struct macroblock_plane *const p = &x->plane[plane];
+ const struct macroblockd_plane *const pd = &x->e_mbd.plane[plane];
+ assert(plane_bsize < BLOCK_SIZES_ALL);
+ const int bw = block_size_wide[plane_bsize];
+ const int bh = block_size_high[plane_bsize];
-void av1_subtract_plane(MACROBLOCK *x, BLOCK_SIZE plane_bsize, int plane,
- bool do_border_pad);
+ av1_subtract_block(x, bh, bw, p->src_diff, bw, p->src.buf, p->src.stride,
+ pd->dst.buf, pd->dst.stride, plane, plane_bsize, 0, 0,
+ DCT_DCT, do_border_pad);
+}
static inline void av1_set_txb_context(MACROBLOCK *x, int plane, int block,
TX_SIZE tx_size, ENTROPY_CONTEXT *a,
diff --git a/av1/encoder/encodetxb.h b/av1/encoder/encodetxb.h
index ec8c457d0c..b2bf71e2b2 100644
--- a/av1/encoder/encodetxb.h
+++ b/av1/encoder/encodetxb.h
@@ -125,30 +125,7 @@ void av1_write_intra_coeffs_mb(const AV1_COMMON *const cm, MACROBLOCK *x,
*/
static inline uint8_t av1_get_txb_entropy_context(const tran_low_t *qcoeff,
const SCAN_ORDER *scan_order,
- int eob) {
- if (eob == 0) return 0;
- const int32_t dc_val = qcoeff[0];
- int cul_level = abs(dc_val);
- if (cul_level < COEFF_CONTEXT_MASK && eob > 1) {
- const int16_t *const scan = scan_order->scan;
- int c = 1;
- for (; c + 3 < eob; c += 4) {
- cul_level += abs(qcoeff[scan[c]]) + abs(qcoeff[scan[c + 1]]) +
- abs(qcoeff[scan[c + 2]]) + abs(qcoeff[scan[c + 3]]);
- if (cul_level >= COEFF_CONTEXT_MASK) break;
- }
- if (cul_level < COEFF_CONTEXT_MASK) {
- for (; c < eob; ++c) {
- cul_level += abs(qcoeff[scan[c]]);
- }
- }
- }
-
- cul_level = AOMMIN(COEFF_CONTEXT_MASK, cul_level);
- set_dc_sign(&cul_level, dc_val);
-
- return (uint8_t)cul_level;
-}
+ int eob);
/*!\brief Update the probability model (cdf) and the entropy context related to
* coefficient coding for all transform blocks in the intra macroblock.
diff --git a/av1/encoder/hybrid_fwd_txfm.c b/av1/encoder/hybrid_fwd_txfm.c
index 4b7fec5b27..1138d20fe9 100644
--- a/av1/encoder/hybrid_fwd_txfm.c
+++ b/av1/encoder/hybrid_fwd_txfm.c
@@ -230,14 +230,6 @@ static void highbd_fwd_txfm_64x64(const int16_t *src_diff, tran_low_t *coeff,
av1_fwd_txfm2d_64x64(src_diff, dst_coeff, diff_stride, DCT_DCT, bd);
}
-void av1_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff, int diff_stride,
- TxfmParam *txfm_param) {
- if (txfm_param->bd == 8)
- av1_lowbd_fwd_txfm(src_diff, coeff, diff_stride, txfm_param);
- else
- av1_highbd_fwd_txfm(src_diff, coeff, diff_stride, txfm_param);
-}
-
void av1_lowbd_fwd_txfm_c(const int16_t *src_diff, tran_low_t *coeff,
int diff_stride, TxfmParam *txfm_param) {
av1_highbd_fwd_txfm(src_diff, coeff, diff_stride, txfm_param);
diff --git a/av1/encoder/hybrid_fwd_txfm.h b/av1/encoder/hybrid_fwd_txfm.h
index 0a9081bc4c..32fcc434d9 100644
--- a/av1/encoder/hybrid_fwd_txfm.h
+++ b/av1/encoder/hybrid_fwd_txfm.h
@@ -13,17 +13,25 @@
#define AOM_AV1_ENCODER_HYBRID_FWD_TXFM_H_
#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
+
+#include "av1/common/blockd.h"
#ifdef __cplusplus
extern "C" {
#endif
-void av1_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff, int diff_stride,
- TxfmParam *txfm_param);
-
void av1_highbd_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff,
int diff_stride, TxfmParam *txfm_param);
+static inline void av1_fwd_txfm(const int16_t *src_diff, tran_low_t *coeff,
+ int diff_stride, TxfmParam *txfm_param) {
+ if (LIKELY(txfm_param->bd == 8))
+ av1_lowbd_fwd_txfm(src_diff, coeff, diff_stride, txfm_param);
+ else
+ av1_highbd_fwd_txfm(src_diff, coeff, diff_stride, txfm_param);
+}
+
/*!\brief Apply Hadamard or DCT transform
*
* \callergraph
diff --git a/av1/encoder/txb_rdopt.c b/av1/encoder/txb_rdopt.c
index 8384ee4a4a..576052d35b 100644
--- a/av1/encoder/txb_rdopt.c
+++ b/av1/encoder/txb_rdopt.c
@@ -541,16 +541,19 @@ static AOM_FORCE_INLINE void update_coeff_simple_facade(
*si = 0;
}
-int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
- int block, TX_SIZE tx_size, TX_TYPE tx_type,
- const TXB_CTX *const txb_ctx, int *rate_cost,
- int sharpness) {
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
+static int av1_optimize_txb_multi(const struct AV1_COMP *cpi, MACROBLOCK *x,
+ int plane, int block, TX_SIZE tx_size,
+ TX_TYPE tx_type, const TXB_CTX *const txb_ctx,
+ int *rate_cost, int sharpness, int eob,
+ int64_t rdmult, int is_noise_pattern) {
MACROBLOCKD *xd = &x->e_mbd;
const struct macroblock_plane *p = &x->plane[plane];
const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type);
const int16_t *scan = scan_order->scan;
const int shift = av1_get_tx_scale(tx_size);
- int eob = p->eobs[block];
const int16_t *dequant = p->dequant_QTX;
const qm_val_t *iqmatrix =
tx_type < IDTX
@@ -567,66 +570,24 @@ int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
const tran_low_t *tcoeff = p->coeff + block_offset;
const CoeffCosts *coeff_costs = &x->coeff_costs;
- // This function is not called if eob = 0.
- assert(eob > 0);
-
const AV1_COMMON *cm = &cpi->common;
const PLANE_TYPE plane_type = get_plane_type(plane);
const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
const TX_CLASS tx_class = tx_type_to_class[tx_type];
- const MB_MODE_INFO *mbmi = xd->mi[0];
const int bhl = get_txb_bhl(tx_size);
const int width = get_txb_wide(tx_size);
const int height = get_txb_high(tx_size);
assert(height == (1 << bhl));
- const int is_inter = is_inter_block(mbmi);
const LV_MAP_COEFF_COST *txb_costs =
&coeff_costs->coeff_costs[txs_ctx][plane_type];
const int eob_multi_size = txsize_log2_minus4[tx_size];
const LV_MAP_EOB_COST *txb_eob_costs =
&coeff_costs->eob_costs[eob_multi_size][plane_type];
- int is_noise_pattern = 0;
-#if CONFIG_AV1_HIGHBITDEPTH
- if (xd->bd > 8 && sharpness == 3 && plane == AOM_PLANE_Y) {
- int64_t src_var, rec_var;
- av1_get_variance_stats(x, &src_var, &rec_var);
- if (src_var > rec_var) {
- const int num_pixels =
- block_size_wide[mbmi->bsize] * block_size_high[mbmi->bsize];
- const int64_t src_var_per_px = src_var / num_pixels;
- if (src_var_per_px < 64) {
- is_noise_pattern = 1;
- }
- }
- }
-#endif
-
- // For the IQ and SSIMULACRA 2 tunings, increase rshift from 2 to 4.
- // This biases trellis quantization towards keeping more coefficients, and
- // together with the IQ and SSIMULACRA2 rdmult adjustment in
- // av1_compute_rd_mult_based_on_qindex(), this helps preserve image
- // features (like repeating patterns and camera noise/film grain), which
- // improves SSIMULACRA 2 scores.
- const int rshift = (cpi->oxcf.tune_cfg.tuning == AOM_TUNE_IQ ||
- cpi->oxcf.tune_cfg.tuning == AOM_TUNE_SSIMULACRA2)
- ? 7
- : (is_noise_pattern ? 6 : 5);
-
- const int (*trellis_rd_mult)[2] = cpi->sf.tx_sf.use_chroma_trellis_rd_mult
- ? plane_rd_mult_chroma
- : plane_rd_mult;
- const int64_t rdmult = ROUND_POWER_OF_TWO(
- (int64_t)x->rdmult * (8 - sharpness) *
- (trellis_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8))),
- rshift);
-
uint8_t levels_buf[TX_PAD_2D];
uint8_t *const levels = set_levels(levels_buf, height);
- if (eob > 1) av1_txb_init_levels(qcoeff, width, height, levels);
-
- // TODO(angirbird): check iqmatrix
+ av1_txb_init_levels(qcoeff, width, height, levels);
const int non_skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][0];
const int skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1];
@@ -730,6 +691,175 @@ int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
return eob;
}
+int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
+ int block, TX_SIZE tx_size, TX_TYPE tx_type,
+ const TXB_CTX *const txb_ctx, int *rate_cost,
+ int sharpness) {
+ MACROBLOCKD *xd = &x->e_mbd;
+ const struct macroblock_plane *p = &x->plane[plane];
+ const int eob = p->eobs[block];
+ assert(eob > 0);
+
+ const PLANE_TYPE plane_type = get_plane_type(plane);
+ const MB_MODE_INFO *mbmi = xd->mi[0];
+ const int is_inter = is_inter_block(mbmi);
+
+ int is_noise_pattern = 0;
+#if CONFIG_AV1_HIGHBITDEPTH
+ if (xd->bd > 8 && sharpness == 3 && plane == AOM_PLANE_Y) {
+ int64_t src_var, rec_var;
+ av1_get_variance_stats(x, &src_var, &rec_var);
+ if (src_var > rec_var) {
+ const int num_pixels =
+ block_size_wide[mbmi->bsize] * block_size_high[mbmi->bsize];
+ const int64_t src_var_per_px = src_var / num_pixels;
+ if (src_var_per_px < 64) {
+ is_noise_pattern = 1;
+ }
+ }
+ }
+#endif
+
+ const int rshift = (cpi->oxcf.tune_cfg.tuning == AOM_TUNE_IQ ||
+ cpi->oxcf.tune_cfg.tuning == AOM_TUNE_SSIMULACRA2)
+ ? 7
+ : (is_noise_pattern ? 6 : 5);
+
+ const int (*trellis_rd_mult)[2] = cpi->sf.tx_sf.use_chroma_trellis_rd_mult
+ ? plane_rd_mult_chroma
+ : plane_rd_mult;
+ const int64_t rdmult = ROUND_POWER_OF_TWO(
+ (int64_t)x->rdmult * (8 - sharpness) *
+ (trellis_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8))),
+ rshift);
+
+ if (eob > 1) {
+ return av1_optimize_txb_multi(cpi, x, plane, block, tx_size, tx_type,
+ txb_ctx, rate_cost, sharpness, eob, rdmult,
+ is_noise_pattern);
+ }
+
+ // Fast path for DC-only blocks (eob == 1): evaluate DC level reduction
+ // without initializing the 2D padded levels buffer.
+ const int block_offset = BLOCK_OFFSET(block);
+ tran_low_t *qcoeff = p->qcoeff + block_offset;
+ tran_low_t *dqcoeff = p->dqcoeff + block_offset;
+ const CoeffCosts *coeff_costs = &x->coeff_costs;
+ const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
+ const TX_CLASS tx_class = tx_type_to_class[tx_type];
+ const LV_MAP_COEFF_COST *txb_costs =
+ &coeff_costs->coeff_costs[txs_ctx][plane_type];
+ const int eob_multi_size = txsize_log2_minus4[tx_size];
+ const LV_MAP_EOB_COST *txb_eob_costs =
+ &coeff_costs->eob_costs[eob_multi_size][plane_type];
+
+ const int non_skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][0];
+ const int skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1];
+ const int eob_cost = txb_eob_costs->eob_cost[tx_class != TX_CLASS_2D][0];
+ const tran_low_t qc = qcoeff[0];
+ const tran_low_t abs_qc = abs(qc);
+ const int sign = qc < 0;
+ int accu_rate = eob_cost;
+
+ if (abs_qc == 1) {
+ accu_rate += txb_costs->base_eob_cost[0][0] +
+ txb_costs->dc_sign_cost[txb_ctx->dc_sign_ctx][sign];
+ if (sharpness == 0) {
+ const tran_low_t tqc = p->coeff[block_offset];
+ const tran_low_t dqc = dqcoeff[0];
+ const int shift = av1_get_tx_scale(tx_size);
+ const qm_val_t *qmatrix =
+ (cpi->oxcf.tune_cfg.dist_metric == AOM_DIST_METRIC_QM_PSNR &&
+ tx_type < IDTX)
+ ? xd->plane[plane].seg_qmatrix[mbmi->segment_id][tx_size]
+ : NULL;
+ const int64_t dist = get_coeff_dist(tqc, dqc, shift, qmatrix, 0);
+ const int64_t dist0 = get_coeff_dist(tqc, 0, shift, qmatrix, 0);
+ const int64_t accu_dist = dist - dist0;
+ if (RDCOST(rdmult, skip_cost, 0) <
+ RDCOST(rdmult, accu_rate + non_skip_cost, accu_dist)) {
+ qcoeff[0] = 0;
+ dqcoeff[0] = 0;
+ p->eobs[block] = 0;
+ p->txb_entropy_ctx[block] = 0;
+ *rate_cost = skip_cost;
+ return 0;
+ }
+ }
+ } else {
+ const tran_low_t tqc = p->coeff[block_offset];
+ const tran_low_t dqc = dqcoeff[0];
+ const int shift = av1_get_tx_scale(tx_size);
+ const qm_val_t *iqmatrix =
+ tx_type < IDTX
+ ? xd->plane[plane].seg_iqmatrix[mbmi->segment_id][tx_size]
+ : NULL;
+ const qm_val_t *qmatrix =
+ (cpi->oxcf.tune_cfg.dist_metric == AOM_DIST_METRIC_QM_PSNR &&
+ tx_type < IDTX)
+ ? xd->plane[plane].seg_qmatrix[mbmi->segment_id][tx_size]
+ : NULL;
+ const int dc_sign_cost =
+ txb_costs->dc_sign_cost[txb_ctx->dc_sign_ctx][sign];
+ int rate =
+ txb_costs->base_eob_cost[0][AOMMIN(abs_qc, 3) - 1] + dc_sign_cost;
+ if (abs_qc > NUM_BASE_LEVELS) {
+ rate += get_br_cost(abs_qc, txb_costs->lps_cost[0]);
+ }
+ const int dqv = get_dqv(p->dequant_QTX, 0, iqmatrix);
+ tran_low_t qc_low, dqc_low;
+ get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low);
+ const tran_low_t abs_qc_low = abs_qc - 1;
+ int rate_low =
+ txb_costs->base_eob_cost[0][AOMMIN(abs_qc_low, 3) - 1] + dc_sign_cost;
+ if (abs_qc_low > NUM_BASE_LEVELS) {
+ rate_low += get_br_cost(abs_qc_low, txb_costs->lps_cost[0]);
+ }
+ int64_t dist_diff_0, dist_diff_low_0;
+ if (qmatrix == NULL) {
+ const int64_t tqc2 = (int64_t)tqc * 2;
+ dist_diff_0 = ((int64_t)dqc * (dqc - tqc2)) * (1LL << (2 * shift));
+ dist_diff_low_0 =
+ ((int64_t)dqc_low * (dqc_low - tqc2)) * (1LL << (2 * shift));
+ } else {
+ const int64_t qm_val = qmatrix[0];
+ const int64_t dist0 = get_coeff_dist_qm(tqc, 0, shift, qm_val);
+ dist_diff_0 = get_coeff_dist_qm(tqc, dqc, shift, qm_val) - dist0;
+ dist_diff_low_0 = get_coeff_dist_qm(tqc, dqc_low, shift, qm_val) - dist0;
+ }
+ const int64_t rd = RDCOST(rdmult, rate, dist_diff_0);
+ const int64_t rd_low = RDCOST(rdmult, rate_low, dist_diff_low_0);
+ int64_t accu_dist;
+ if (rd_low < rd) {
+ qcoeff[0] = qc_low;
+ dqcoeff[0] = dqc_low;
+ accu_rate += rate_low;
+ accu_dist = dist_diff_low_0;
+ } else {
+ accu_rate += rate;
+ accu_dist = dist_diff_0;
+ }
+ if (sharpness == 0) {
+ if (RDCOST(rdmult, skip_cost, 0) <
+ RDCOST(rdmult, accu_rate + non_skip_cost, accu_dist)) {
+ qcoeff[0] = 0;
+ dqcoeff[0] = 0;
+ p->eobs[block] = 0;
+ p->txb_entropy_ctx[block] = 0;
+ *rate_cost = skip_cost;
+ return 0;
+ }
+ }
+ }
+
+ const int tx_type_cost = get_tx_type_cost(
+ x, xd, plane, tx_size, tx_type, cpi->common.features.reduced_tx_set_used);
+ *rate_cost = accu_rate + non_skip_cost + tx_type_cost;
+ p->eobs[block] = 1;
+ p->txb_entropy_ctx[block] = av1_get_txb_entropy_context(qcoeff, NULL, 1);
+ return 1;
+}
+
static AOM_FORCE_INLINE int warehouse_efficients_txb_loop(
const tran_low_t *const __restrict qcoeff,
const int16_t *const __restrict scan, const int eob,
diff --git a/av1/encoder/txb_rdopt.h b/av1/encoder/txb_rdopt.h
index c5b6abcb11..96ebe1f687 100644
--- a/av1/encoder/txb_rdopt.h
+++ b/av1/encoder/txb_rdopt.h
@@ -15,6 +15,7 @@
#include "av1/common/blockd.h"
#include "av1/common/txb_common.h"
#include "av1/encoder/encoder.h"
+#include "av1/encoder/encodetxb.h"
#ifdef __cplusplus
extern "C" {
@@ -55,6 +56,25 @@ int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
const TXB_CTX *const txb_ctx, int *rate_cost,
int sharpness);
+static inline int av1_optimize_b(const struct AV1_COMP *cpi, MACROBLOCK *x,
+ int plane, int block, TX_SIZE tx_size,
+ TX_TYPE tx_type, const TXB_CTX *const txb_ctx,
+ int *rate_cost) {
+ MACROBLOCKD *const xd = &x->e_mbd;
+ struct macroblock_plane *const p = &x->plane[plane];
+ const int eob = p->eobs[block];
+ const int segment_id = xd->mi[0]->segment_id;
+
+ if (eob == 0 || !cpi->optimize_seg_arr[segment_id] ||
+ xd->lossless[segment_id]) {
+ *rate_cost = av1_cost_skip_txb(&x->coeff_costs, txb_ctx, plane, tx_size);
+ return eob;
+ }
+
+ return av1_optimize_txb(cpi, x, plane, block, tx_size, tx_type, txb_ctx,
+ rate_cost, cpi->oxcf.algo_cfg.sharpness);
+}
+
/*!\brief Compute the entropy cost of coding coefficients in a transform block.
*
* \ingroup coefficient_coding
diff --git a/av1/encoder/x86/av1_fwd_txfm2d_avx2.c b/av1/encoder/x86/av1_fwd_txfm2d_avx2.c
index 081f22058e..88599d0bbc 100644
--- a/av1/encoder/x86/av1_fwd_txfm2d_avx2.c
+++ b/av1/encoder/x86/av1_fwd_txfm2d_avx2.c
@@ -1810,22 +1810,16 @@ static inline void store_buffer_16bit_to_32bit_w8_avx2(const __m128i *const in,
}
}
-static void av1_lowbd_fwd_txfm2d_8x8_avx2(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
+static void lowbd_fwd_txfm2d_8x8_non_dct_avx2(const int16_t *input,
+ int32_t *output, int stride,
+ TX_TYPE tx_type, int bd) {
(void)bd;
__m128i buf0[8], buf1[8];
const int8_t *shift = av1_fwd_txfm_shift_ls[TX_8X8];
- // Fast path for DCT_DCT: bypass function pointer table and flip checks.
- if (tx_type == DCT_DCT) {
- load_buffer_and_round_shift(input, stride, buf0, shift[0]);
- fdct8x8_new_sse2(buf0, buf0, 13);
- transpose_round_shift_8x8(buf0, buf1, shift[1]);
- fdct8x8_new_sse2(buf1, buf1, 13);
- store_buffer_16bit_to_32bit_w8_avx2(buf1, output, 8, 8);
- return;
- }
-
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
assert(shift[0] > 0);
@@ -1854,32 +1848,38 @@ static void av1_lowbd_fwd_txfm2d_8x8_avx2(const int16_t *input, int32_t *output,
case FLIPADST_1D: fadst8x8_new_sse2(buf1, buf1, 13); break;
default: fidentity8x8_new_sse2(buf1, buf1, 13); break;
}
-
assert(shift[2] == 0);
store_buffer_16bit_to_32bit_w8_avx2(buf1, output, 8, 8);
}
-static void lowbd_fwd_txfm2d_16x16_avx2(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+static void av1_lowbd_fwd_txfm2d_8x8_avx2(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
+ if (UNLIKELY(tx_type != DCT_DCT)) {
+ lowbd_fwd_txfm2d_8x8_non_dct_avx2(input, output, stride, tx_type, bd);
+ return;
+ }
+ (void)bd;
+ __m128i buf0[8], buf1[8];
+ const int8_t *shift = av1_fwd_txfm_shift_ls[TX_8X8];
+ load_buffer_and_round_shift(input, stride, buf0, shift[0]);
+ fdct8x8_new_sse2(buf0, buf0, 13);
+ transpose_round_shift_8x8(buf0, buf1, shift[1]);
+ fdct8x8_new_sse2(buf1, buf1, 13);
+ store_buffer_16bit_to_32bit_w8_avx2(buf1, output, 8, 8);
+}
+
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
+static void lowbd_fwd_txfm2d_16x16_non_dct_avx2(const int16_t *input,
+ int32_t *output, int stride,
+ TX_TYPE tx_type, int bd) {
(void)bd;
const TX_SIZE tx_size = TX_16X16;
__m256i buf0[16], buf1[16];
const int8_t *shift = av1_fwd_txfm_shift_ls[tx_size];
const int width = tx_size_wide[tx_size];
const int height = tx_size_high[tx_size];
-
- // Fast path for DCT_DCT: inline 16x16 DCT with constant cos_bit (13, 12).
- if (tx_type == DCT_DCT) {
- load_buffer_16bit_to_16bit_avx2(input, stride, buf0, height);
- round_shift_16bit_w16_avx2(buf0, height, shift[0]);
- fdct16x16_new_avx2(buf0, buf0, 13);
- round_shift_16bit_w16_avx2(buf0, height, shift[1]);
- transpose_16bit_16x16_avx2(buf0, buf1);
- fdct16x16_new_avx2(buf1, buf1, 12);
- store_buffer_16bit_to_32bit_w16_avx2(buf1, output, height, width);
- return;
- }
-
const int txw_idx = get_txw_idx(tx_size);
const int txh_idx = get_txh_idx(tx_size);
const int cos_bit_col = av1_fwd_cos_bit_col[txw_idx][txh_idx];
@@ -1912,6 +1912,28 @@ static void lowbd_fwd_txfm2d_16x16_avx2(const int16_t *input, int32_t *output,
store_buffer_16bit_to_32bit_w16_avx2(buf, output + i * 16, height, width);
}
+static void lowbd_fwd_txfm2d_16x16_avx2(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
+ if (UNLIKELY(tx_type != DCT_DCT)) {
+ lowbd_fwd_txfm2d_16x16_non_dct_avx2(input, output, stride, tx_type, bd);
+ return;
+ }
+ (void)bd;
+ const TX_SIZE tx_size = TX_16X16;
+ __m256i buf0[16], buf1[16];
+ const int8_t *shift = av1_fwd_txfm_shift_ls[tx_size];
+ const int width = tx_size_wide[tx_size];
+ const int height = tx_size_high[tx_size];
+
+ load_buffer_16bit_to_16bit_avx2(input, stride, buf0, height);
+ round_shift_16bit_w16_avx2(buf0, height, shift[0]);
+ fdct16x16_new_avx2(buf0, buf0, 13);
+ round_shift_16bit_w16_avx2(buf0, height, shift[1]);
+ transpose_16bit_16x16_avx2(buf0, buf1);
+ fdct16x16_new_avx2(buf1, buf1, 12);
+ store_buffer_16bit_to_32bit_w16_avx2(buf1, output, height, width);
+}
+
static void lowbd_fwd_txfm2d_32x32_avx2(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
(void)bd;
@@ -2975,8 +2997,12 @@ static const transform_1d_sse2 row_txfm16x8_arr[TX_TYPES] = {
fadst8x16_new_avx2 // H_FLIPADST
};
-static void lowbd_fwd_txfm2d_8x16_avx2(const int16_t *input, int32_t *output,
- int stride, TX_TYPE tx_type, int bd) {
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
+static void lowbd_fwd_txfm2d_8x16_non_dct_avx2(const int16_t *input,
+ int32_t *output, int stride,
+ TX_TYPE tx_type, int bd) {
(void)bd;
__m128i buf0[16], buf1[16];
__m256i buf2[8];
@@ -3019,8 +3045,33 @@ static void lowbd_fwd_txfm2d_8x16_avx2(const int16_t *input, int32_t *output,
store_rect_buffer_16bit_to_32bit_w16_avx2(buf2, output, height, width);
}
-static void lowbd_fwd_txfm2d_16x8_avx2(const int16_t *input, int32_t *output,
+static void lowbd_fwd_txfm2d_8x16_avx2(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
+ if (UNLIKELY(tx_type != DCT_DCT)) {
+ lowbd_fwd_txfm2d_8x16_non_dct_avx2(input, output, stride, tx_type, bd);
+ return;
+ }
+ (void)bd;
+ __m128i buf0[16], buf1[16];
+ __m256i buf2[8];
+ const int8_t *shift = av1_fwd_txfm_shift_ls[TX_8X16];
+ load_buffer_16bit_to_16bit(input, stride, buf0, 16);
+ round_shift_16bit(buf0, 16, shift[0]);
+ fdct8x16_new_avx2(buf0, buf0, 13);
+ round_shift_16bit(buf0, 16, shift[1]);
+ transpose_16bit_8x8(buf0, buf1);
+ transpose_16bit_8x8(buf0 + 8, buf1 + 8);
+ pack_reg(buf1, buf1 + 8, buf2);
+ fdct8x8_new_avx2(buf2, buf2, 13);
+ store_rect_buffer_16bit_to_32bit_w16_avx2(buf2, output, 16, 8);
+}
+
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
+static void lowbd_fwd_txfm2d_16x8_non_dct_avx2(const int16_t *input,
+ int32_t *output, int stride,
+ TX_TYPE tx_type, int bd) {
(void)bd;
__m128i buf0[16], buf1[16];
__m256i buf2[8];
@@ -3063,6 +3114,28 @@ static void lowbd_fwd_txfm2d_16x8_avx2(const int16_t *input, int32_t *output,
store_rect_buffer_16bit_to_32bit_w8(buf, output, height, width);
}
+static void lowbd_fwd_txfm2d_16x8_avx2(const int16_t *input, int32_t *output,
+ int stride, TX_TYPE tx_type, int bd) {
+ if (UNLIKELY(tx_type != DCT_DCT)) {
+ lowbd_fwd_txfm2d_16x8_non_dct_avx2(input, output, stride, tx_type, bd);
+ return;
+ }
+ (void)bd;
+ __m128i buf0[16], buf1[16];
+ __m256i buf2[8];
+ const int8_t *shift = av1_fwd_txfm_shift_ls[TX_16X8];
+ load_buffer_16bit_to_16bit(input + 8 * 0, stride, buf0, 8);
+ load_buffer_16bit_to_16bit(input + 8 * 1, stride, &buf0[8], 8);
+ pack_reg(buf0, &buf0[8], buf2);
+ round_shift_16bit_w16_avx2(buf2, 8, shift[0]);
+ fdct8x8_new_avx2(buf2, buf2, 13);
+ round_shift_16bit_w16_avx2(buf2, 8, shift[1]);
+ transpose_16bit_16x8_avx2(buf2, buf2);
+ extract_reg(buf2, buf1);
+ fdct8x16_new_avx2(buf1, buf1, 13);
+ store_rect_buffer_16bit_to_32bit_w8(buf1, output, 8, 16);
+}
+
static FwdTxfm2dFunc fwd_txfm2d_func_ls[TX_SIZES_ALL] = {
av1_lowbd_fwd_txfm2d_4x4_sse2, // 4x4 transform
av1_lowbd_fwd_txfm2d_8x8_avx2, // 8x8 transform