Commit c7a9274c1f for aom
commit c7a9274c1f12de61126daf4e15c9f24efaa2ca6f
Author: Frank Galligan <fgalligan@google.com>
Date: Fri Oct 30 12:07:34 2026 -0700
av1: Optimize update_coeff_simple and tx search
Split out non-zero coefficient handling into update_coeff_simple_nz so
the qc == 0 scan loop stays register-resident, inline
av1_get_txb_entropy_context and get_tx_type_cost, add an eob == 1 fast
path in encodetxb.c, and fast-path single-candidate transform search in
search_tx_type.
Benchmarking results (incremental vs prev CL / cumulative vs baseline):
- Stock avifenc still-image (Default 1T): -0.44% / -21.36% encode time
- Stock avifenc still-image (Default 8T): -1.16% / -17.22% encode time
- 48-image multi-rate corpus (Full Suite 1T): -0.25% / -21.62% time
- Bit-exact output (1T and 8T)
TAG=agy
CONV=b929a630-a6cd-45a0-979c-1767c30178ee
Change-Id: I951e4e65a270dedc05eb2ac6904c4889af303461
diff --git a/av1/common/txb_common.h b/av1/common/txb_common.h
index 84abf4faac..372d7cd6de 100644
--- a/av1/common/txb_common.h
+++ b/av1/common/txb_common.h
@@ -86,7 +86,8 @@ static inline int get_br_ctx_2d(const uint8_t *const levels,
const int row = c - (col << bhl);
const intptr_t stride = (1 << bhl) + TX_PAD_HOR;
const uint8_t *const lv = levels + c + (col << TX_PAD_HOR_LOG2);
- int mag = lv[1] + lv[stride] + lv[stride + 1];
+ const uint8_t *const lvs = lv + stride;
+ int mag = lv[1] + lvs[0] + lvs[1];
mag = AOMMIN((mag + 1) >> 1, 6);
//((row | col) < 2) is equivalent to ((row < 2) && (col < 2))
if ((row | col) < 2) return mag + 7;
@@ -113,16 +114,17 @@ static AOM_FORCE_INLINE int get_br_ctx(const uint8_t *const levels,
const int row = c - (col << bhl);
const intptr_t stride = (1 << bhl) + TX_PAD_HOR;
const uint8_t *const lv = levels + c + (col << TX_PAD_HOR_LOG2);
- int mag = lv[1] + lv[stride];
+ const uint8_t *const lvs = lv + stride;
+ int mag = lv[1] + lvs[0];
switch (tx_class) {
case TX_CLASS_2D:
- mag += lv[stride + 1];
+ mag += lvs[1];
mag = AOMMIN((mag + 1) >> 1, 6);
if (c == 0) return mag;
if ((row | col) < 2) return mag + 7;
break;
case TX_CLASS_HORIZ:
- mag += lv[stride << 1];
+ mag += lvs[stride];
mag = AOMMIN((mag + 1) >> 1, 6);
if (c == 0) return mag;
if (col == 0) return mag + 7;
@@ -148,14 +150,15 @@ static AOM_FORCE_INLINE int get_br_ctx_simple(const uint8_t *const levels,
const int row = c - (col << bhl);
const intptr_t stride = (1 << bhl) + TX_PAD_HOR;
const uint8_t *const lv = levels + c + (col << TX_PAD_HOR_LOG2);
- int mag = lv[1] + lv[stride];
+ const uint8_t *const lvs = lv + stride;
+ int mag = lv[1] + lvs[0];
switch (tx_class) {
case TX_CLASS_2D:
- mag += lv[stride + 1];
+ mag += lvs[1];
mag = AOMMIN((mag + 1) >> 1, 6);
return mag + (((row | col) < 2) ? 7 : 14);
case TX_CLASS_HORIZ:
- mag += lv[stride << 1];
+ mag += lvs[stride];
mag = AOMMIN((mag + 1) >> 1, 6);
return mag + ((col == 0) ? 7 : 14);
case TX_CLASS_VERT:
@@ -183,23 +186,25 @@ static AOM_FORCE_INLINE int get_nz_mag(const uint8_t *const levels,
const int bhl, const TX_CLASS tx_class) {
int mag;
const ptrdiff_t stride = (1 << bhl) + TX_PAD_HOR;
+ const uint8_t *const lvs = levels + stride;
// Note: AOMMIN(level, 3) is useless for decoder since level < 3.
- mag = clip_max3[levels[stride]]; // { 0, 1 }
- mag += clip_max3[levels[1]]; // { 1, 0 }
+ mag = clip_max3[lvs[0]]; // { 0, 1 }
+ mag += clip_max3[levels[1]]; // { 1, 0 }
if (tx_class == TX_CLASS_2D) {
- mag += clip_max3[levels[stride + 1]]; // { 1, 1 }
- mag += clip_max3[levels[stride * 2]]; // { 0, 2 }
- mag += clip_max3[levels[2]]; // { 2, 0 }
+ mag += clip_max3[lvs[1]]; // { 1, 1 }
+ mag += clip_max3[lvs[stride]]; // { 0, 2 }
+ mag += clip_max3[levels[2]]; // { 2, 0 }
} else if (tx_class == TX_CLASS_VERT) {
mag += clip_max3[levels[2]]; // { 2, 0 }
mag += clip_max3[levels[3]]; // { 3, 0 }
mag += clip_max3[levels[4]]; // { 4, 0 }
} else {
- mag += clip_max3[levels[stride * 2]]; // { 0, 2 }
- mag += clip_max3[levels[stride * 3]]; // { 0, 3 }
- mag += clip_max3[levels[stride * 4]]; // { 0, 4 }
+ const uint8_t *const lvs2 = lvs + stride;
+ mag += clip_max3[lvs2[0]]; // { 0, 2 }
+ mag += clip_max3[lvs2[stride]]; // { 0, 3 }
+ mag += clip_max3[lvs2[stride * 2]]; // { 0, 4 }
}
return mag;
@@ -358,10 +363,7 @@ static inline int get_lower_levels_ctx_general(int is_last, int scan_idx,
}
static inline void set_dc_sign(int *cul_level, int dc_val) {
- if (dc_val < 0)
- *cul_level |= 1 << COEFF_CONTEXT_BITS;
- else if (dc_val > 0)
- *cul_level += 2 << COEFF_CONTEXT_BITS;
+ *cul_level += ((dc_val < 0) + ((dc_val > 0) << 1)) << COEFF_CONTEXT_BITS;
}
#define MAX_TX_SIZE_UNIT 16
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 51a0fd16c7..baedd1320d 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -358,11 +358,16 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCK *const x,
const tran_low_t *tcoeff_txb =
cb_coef_buff->tcoeff[plane] + x->mbmi_ext_frame->cb_offset[plane_type];
const tran_low_t *tcoeff = tcoeff_txb + BLOCK_OFFSET(block);
- av1_txb_init_levels(tcoeff, width, height, levels);
const SCAN_ORDER *const scan_order = get_scan(tx_size, tx_type);
const int16_t *const scan = scan_order->scan;
DECLARE_ALIGNED(16, int8_t, coeff_contexts[MAX_TX_SQUARE]);
- av1_get_nz_map_contexts(levels, scan, eob, tx_size, tx_class, coeff_contexts);
+ if (eob > 1) {
+ av1_txb_init_levels(tcoeff, width, height, levels);
+ av1_get_nz_map_contexts(levels, scan, eob, tx_size, tx_class,
+ coeff_contexts);
+ } else {
+ coeff_contexts[0] = 0;
+ }
const int bhl = get_txb_bhl(tx_size);
for (int c = eob - 1; c >= 0; --c) {
@@ -383,7 +388,7 @@ void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCK *const x,
if (level > NUM_BASE_LEVELS) {
// level is above 1.
const int base_range = level - 1 - NUM_BASE_LEVELS;
- const int br_ctx = get_br_ctx(levels, pos, bhl, tx_class);
+ const int br_ctx = (eob > 1) ? get_br_ctx(levels, pos, bhl, tx_class) : 0;
aom_cdf_prob *cdf =
ec_ctx->coeff_br_cdf[AOMMIN(txs_ctx, TX_32X32)][plane_type][br_ctx];
for (int idx = 0; idx < COEFF_BASE_RANGE; idx += BR_CDF_SIZE - 1) {
@@ -458,32 +463,6 @@ void av1_write_intra_coeffs_mb(const AV1_COMMON *const cm, MACROBLOCK *x,
}
}
-uint8_t av1_get_txb_entropy_context(const tran_low_t *qcoeff,
- const SCAN_ORDER *scan_order, int eob) {
- if (eob == 0) return 0;
- const int32_t dc_val = qcoeff[0];
- int cul_level = abs(dc_val);
- if (cul_level < COEFF_CONTEXT_MASK && eob > 1) {
- const int16_t *const __restrict scan = scan_order->scan;
- int c = 1;
- for (; c + 3 < eob; c += 4) {
- cul_level += abs(qcoeff[scan[c]]) + abs(qcoeff[scan[c + 1]]) +
- abs(qcoeff[scan[c + 2]]) + abs(qcoeff[scan[c + 3]]);
- if (cul_level >= COEFF_CONTEXT_MASK) break;
- }
- if (cul_level < COEFF_CONTEXT_MASK) {
- for (; c < eob; ++c) {
- cul_level += abs(qcoeff[scan[c]]);
- }
- }
- }
-
- cul_level = AOMMIN(COEFF_CONTEXT_MASK, cul_level);
- set_dc_sign(&cul_level, dc_val);
-
- return (uint8_t)cul_level;
-}
-
static void update_tx_type_count(const AV1_COMP *cpi, const AV1_COMMON *cm,
MACROBLOCKD *xd, int blk_row, int blk_col,
int plane, TX_SIZE tx_size,
@@ -620,11 +599,14 @@ static void update_and_record_txb_context_output(
tran_low_t *tcoeff_txb =
cb_coef_buff->tcoeff[plane] + x->mbmi_ext_frame->cb_offset[plane_type];
tran_low_t *tcoeff = tcoeff_txb + block_offset;
- memcpy(tcoeff, qcoeff, sizeof(*tcoeff) * seg_eob);
+ if (eob == 1) {
+ tcoeff[0] = qcoeff[0];
+ } else {
+ memcpy(tcoeff, qcoeff, sizeof(*tcoeff) * seg_eob);
+ }
uint8_t levels_buf[TX_PAD_2D];
uint8_t *const levels = set_levels(levels_buf, height);
- av1_txb_init_levels(tcoeff, width, height, levels);
update_tx_type_count(cpi, cm, xd, blk_row, blk_col, plane, tx_size,
td->counts, allow_update_cdf);
@@ -643,7 +625,13 @@ static void update_and_record_txb_context_output(
#endif
DECLARE_ALIGNED(16, int8_t, coeff_contexts[MAX_TX_SQUARE]);
- av1_get_nz_map_contexts(levels, scan, eob, tx_size, tx_class, coeff_contexts);
+ if (eob > 1) {
+ av1_txb_init_levels(tcoeff, width, height, levels);
+ av1_get_nz_map_contexts(levels, scan, eob, tx_size, tx_class,
+ coeff_contexts);
+ } else {
+ coeff_contexts[0] = 0;
+ }
for (int c = eob - 1; c >= 0; --c) {
const int pos = scan[c];
@@ -678,7 +666,7 @@ static void update_and_record_txb_context_output(
}
if (level > NUM_BASE_LEVELS) {
const int base_range = level - 1 - NUM_BASE_LEVELS;
- const int br_ctx = get_br_ctx(levels, pos, bhl, tx_class);
+ const int br_ctx = (eob > 1) ? get_br_ctx(levels, pos, bhl, tx_class) : 0;
for (int idx = 0; idx < COEFF_BASE_RANGE; idx += BR_CDF_SIZE - 1) {
const int k = AOMMIN(base_range - idx, BR_CDF_SIZE - 1);
if (allow_update_cdf) {
@@ -771,7 +759,15 @@ static void record_txb_context_output(int plane, int block, int blk_row,
tran_low_t *tcoeff_txb =
cb_coef_buff->tcoeff[plane] + x->mbmi_ext_frame->cb_offset[plane_type];
tran_low_t *tcoeff = tcoeff_txb + block_offset;
+#if !CONFIG_ENTROPY_STATS
+ if (eob == 1) {
+ tcoeff[0] = qcoeff[0];
+ } else {
+ memcpy(tcoeff, qcoeff, sizeof(*tcoeff) * seg_eob);
+ }
+#else
memcpy(tcoeff, qcoeff, sizeof(*tcoeff) * seg_eob);
+#endif
#if CONFIG_ENTROPY_STATS
uint8_t levels_buf[TX_PAD_2D];
diff --git a/av1/encoder/encodetxb.h b/av1/encoder/encodetxb.h
index 45a7b635df..ec8c457d0c 100644
--- a/av1/encoder/encodetxb.h
+++ b/av1/encoder/encodetxb.h
@@ -123,8 +123,32 @@ void av1_write_intra_coeffs_mb(const AV1_COMMON *const cm, MACROBLOCK *x,
* \param[in] eob The scan index of last non-zero coefficient plus
* one
*/
-uint8_t av1_get_txb_entropy_context(const tran_low_t *qcoeff,
- const SCAN_ORDER *scan_order, int eob);
+static inline uint8_t av1_get_txb_entropy_context(const tran_low_t *qcoeff,
+ const SCAN_ORDER *scan_order,
+ int eob) {
+ if (eob == 0) return 0;
+ const int32_t dc_val = qcoeff[0];
+ int cul_level = abs(dc_val);
+ if (cul_level < COEFF_CONTEXT_MASK && eob > 1) {
+ const int16_t *const scan = scan_order->scan;
+ int c = 1;
+ for (; c + 3 < eob; c += 4) {
+ cul_level += abs(qcoeff[scan[c]]) + abs(qcoeff[scan[c + 1]]) +
+ abs(qcoeff[scan[c + 2]]) + abs(qcoeff[scan[c + 3]]);
+ if (cul_level >= COEFF_CONTEXT_MASK) break;
+ }
+ if (cul_level < COEFF_CONTEXT_MASK) {
+ for (; c < eob; ++c) {
+ cul_level += abs(qcoeff[scan[c]]);
+ }
+ }
+ }
+
+ cul_level = AOMMIN(COEFF_CONTEXT_MASK, cul_level);
+ set_dc_sign(&cul_level, dc_val);
+
+ return (uint8_t)cul_level;
+}
/*!\brief Update the probability model (cdf) and the entropy context related to
* coefficient coding for all transform blocks in the intra macroblock.
diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c
index c41bcbf5ff..a8295a8542 100644
--- a/av1/encoder/tx_search.c
+++ b/av1/encoder/tx_search.c
@@ -2181,7 +2181,7 @@ static void search_tx_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
TxfmParam txfm_param;
QUANT_PARAM quant_param;
- int skip_trellis_based_on_satd[TX_TYPES] = { 0 };
+ int best_skip_trellis = 0;
av1_setup_xform(cm, x, tx_size, DCT_DCT, &txfm_param);
av1_setup_quant(tx_size, !skip_trellis,
skip_trellis ? (USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B
@@ -2191,9 +2191,14 @@ static void search_tx_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
const int use_qmatrix =
av1_use_qmatrix(&cm->quant_params, xd, mbmi->segment_id);
+ int num_tx_candidates = TX_TYPES;
+ if (allowed_tx_mask != 0 && (allowed_tx_mask & (allowed_tx_mask - 1)) == 0) {
+ txk_map[0] = get_msb(allowed_tx_mask);
+ num_tx_candidates = 1;
+ }
uint16_t rem_tx_mask = allowed_tx_mask;
// Iterate through all transform type candidates.
- for (int idx = 0; idx < TX_TYPES && rem_tx_mask != 0; ++idx) {
+ for (int idx = 0; idx < num_tx_candidates && rem_tx_mask != 0; ++idx) {
const TX_TYPE tx_type = (TX_TYPE)txk_map[idx];
if (tx_type == TX_TYPE_INVALID || !check_bit_mask(rem_tx_mask, tx_type))
continue;
@@ -2229,7 +2234,7 @@ static void search_tx_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
}
}
- skip_trellis_based_on_satd[tx_type] = skip_trellis_opt_based_on_satd(
+ const int this_skip_trellis = skip_trellis_opt_based_on_satd(
x, &quant_param, plane, block, tx_size, cpi->oxcf.q_cfg.quant_b_adapt,
qstep, txfm_params->coeff_opt_thresholds[1], skip_trellis, dc_only_blk,
&cur_satd);
@@ -2316,6 +2321,7 @@ static void search_tx_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
best_tx_type = tx_type;
best_txb_ctx = x->plane[plane].txb_entropy_ctx[block];
best_eob = x->plane[plane].eobs[block];
+ best_skip_trellis = this_skip_trellis;
// Swap dqcoeff buffers
tran_low_t *const tmp_dqcoeff = best_dqcoeff;
best_dqcoeff = p->dqcoeff;
@@ -2384,7 +2390,7 @@ static void search_tx_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
if (plane == 0) update_txk_array(xd, blk_row, blk_col, tx_size, best_tx_type);
x->plane[plane].txb_entropy_ctx[block] = best_txb_ctx;
x->plane[plane].eobs[block] = best_eob;
- skip_trellis = skip_trellis_based_on_satd[best_tx_type];
+ skip_trellis = best_skip_trellis;
// Point dqcoeff to the quantized coefficients corresponding to the best
// transform type, then we can skip transform and quantization, e.g. in the
diff --git a/av1/encoder/txb_rdopt.c b/av1/encoder/txb_rdopt.c
index cd94156f87..ed7c059398 100644
--- a/av1/encoder/txb_rdopt.c
+++ b/av1/encoder/txb_rdopt.c
@@ -87,107 +87,117 @@ static AOM_FORCE_INLINE void update_coeff_general(
}
}
-static AOM_FORCE_INLINE int update_coeff_simple(
- int si, int eob, TX_SIZE tx_size, TX_CLASS tx_class, int bhl,
- int64_t rdmult, int shift, const int16_t *dequant, const int16_t *scan,
- const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff,
- tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness,
- const qm_val_t *iqmatrix, const qm_val_t *qmatrix) {
- (void)eob;
- // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0)
- // and not the last (scan_idx != eob - 1)
- assert(si != eob - 1);
- assert(si > 0);
- const int ci = scan[si];
+/*!\cond */
+typedef struct {
+ int64_t rdmult;
+ const LV_MAP_COEFF_COST *txb_costs;
+ const tran_low_t *tcoeff;
+ tran_low_t *qcoeff;
+ tran_low_t *dqcoeff;
+ uint8_t *levels;
+ const qm_val_t *iqmatrix;
+ const qm_val_t *qmatrix;
+ int bhl;
+ int shift;
+ int dequant_ac;
+ int sharpness;
+} TxbSimpleCtx;
+/*!\endcond */
+
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
+static int update_coeff_simple_nz(int ci, tran_low_t qc, int coeff_ctx,
+ TX_CLASS tx_class, const TxbSimpleCtx *ctx) {
assert(ci > 0);
- const tran_low_t qc = qcoeff[ci];
- const int coeff_ctx =
- get_lower_levels_ctx_simple(levels, ci, bhl, tx_size, tx_class);
- if (qc == 0) {
- return txb_costs->base_cost[coeff_ctx][0];
- } else {
- const tran_low_t abs_qc = abs(qc);
- const tran_low_t abs_tqc = abs(tcoeff[ci]);
- const tran_low_t abs_dqc = abs(dqcoeff[ci]);
- if (abs_qc == 1) {
- const int *base_cost = txb_costs->base_cost[coeff_ctx];
- const int rate = base_cost[1] + av1_cost_literal(1);
- if (abs_dqc < abs_tqc) {
- return rate;
- }
-
- const int allow_lower_qc = sharpness ? 0 : 1;
- if (allow_lower_qc) {
- int64_t dist_diff_0;
- if (qmatrix == NULL) {
- dist_diff_0 =
- ((int64_t)abs_dqc * (abs_dqc - ((int64_t)abs_tqc * 2))) *
- (1LL << (2 * shift));
- } else {
- const int64_t qm_val = qmatrix[ci];
- const int64_t dist0 = get_coeff_dist_qm(abs_tqc, 0, shift, qm_val);
- dist_diff_0 =
- get_coeff_dist_qm(abs_tqc, abs_dqc, shift, qm_val) - dist0;
- }
- const int rate_low = rate - base_cost[5];
- const int64_t rd = RDCOST(rdmult, rate, dist_diff_0);
- const int64_t rd_low = RDCOST(rdmult, rate_low, 0);
- if (rd_low < rd) {
- qcoeff[ci] = 0;
- dqcoeff[ci] = 0;
- levels[get_padded_idx(ci, bhl)] = 0;
- return rate_low;
- }
- }
+ const LV_MAP_COEFF_COST *txb_costs = ctx->txb_costs;
+ const int bhl = ctx->bhl;
+ const int64_t rdmult = ctx->rdmult;
+ const int shift = ctx->shift;
+ const int sharpness = ctx->sharpness;
+ const qm_val_t *qmatrix = ctx->qmatrix;
+ uint8_t *levels = ctx->levels;
+ const tran_low_t abs_qc = abs(qc);
+ const tran_low_t abs_tqc = abs(ctx->tcoeff[ci]);
+ const tran_low_t abs_dqc = abs(ctx->dqcoeff[ci]);
+ if (abs_qc == 1) {
+ const int *base_cost = txb_costs->base_cost[coeff_ctx];
+ const int rate = base_cost[1] + av1_cost_literal(1);
+ if (abs_dqc < abs_tqc) {
return rate;
- } else {
- if (abs_dqc < abs_tqc) {
- int rate = txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)] +
- av1_cost_literal(1);
- if (abs_qc > NUM_BASE_LEVELS) {
- const int br_ctx = get_br_ctx_simple(levels, ci, bhl, tx_class);
- rate += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]);
- }
- return rate;
- }
+ }
- int rate_low = 0;
- const int rate = get_two_coeff_cost_simple(
- ci, abs_qc, coeff_ctx, txb_costs, bhl, tx_class, levels, &rate_low);
- // update_coeff_simple is only invoked for AC coefficients (si > 0), so
- // dequant[1] can be used directly without a (ci > 0) branch.
- int dqv = dequant[1];
- if (iqmatrix != NULL) {
- dqv = ((iqmatrix[ci] * dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
- }
- const tran_low_t abs_qc_low = abs_qc - 1;
- const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift;
- int64_t dist_diff_0, dist_diff_low_0;
+ const int allow_lower_qc = sharpness ? 0 : 1;
+ if (allow_lower_qc) {
+ int64_t dist_diff_0;
if (qmatrix == NULL) {
- const int64_t abs_tqc2 = (int64_t)abs_tqc << 1;
- dist_diff_0 =
- ((int64_t)abs_dqc * (abs_dqc - abs_tqc2)) * (1LL << (2 * shift));
- dist_diff_low_0 = ((int64_t)abs_dqc_low * (abs_dqc_low - abs_tqc2)) *
- (1LL << (2 * shift));
+ dist_diff_0 = ((int64_t)abs_dqc * (abs_dqc - ((int64_t)abs_tqc * 2))) *
+ (1LL << (2 * shift));
} else {
const int64_t qm_val = qmatrix[ci];
const int64_t dist0 = get_coeff_dist_qm(abs_tqc, 0, shift, qm_val);
dist_diff_0 =
get_coeff_dist_qm(abs_tqc, abs_dqc, shift, qm_val) - dist0;
- dist_diff_low_0 =
- get_coeff_dist_qm(abs_tqc, abs_dqc_low, shift, qm_val) - dist0;
}
+ const int rate_low = rate - base_cost[5];
const int64_t rd = RDCOST(rdmult, rate, dist_diff_0);
- const int64_t rd_low = RDCOST(rdmult, rate_low, dist_diff_low_0);
+ const int64_t rd_low = RDCOST(rdmult, rate_low, 0);
if (rd_low < rd) {
- const int sign = (qc < 0) ? 1 : 0;
- qcoeff[ci] = (-sign ^ abs_qc_low) + sign;
- dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign;
- levels[get_padded_idx(ci, bhl)] = AOMMIN(abs_qc_low, INT8_MAX);
+ ctx->qcoeff[ci] = 0;
+ ctx->dqcoeff[ci] = 0;
+ levels[get_padded_idx(ci, bhl)] = 0;
return rate_low;
}
+ }
+ return rate;
+ } else {
+ if (abs_dqc < abs_tqc) {
+ int rate = txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)] +
+ av1_cost_literal(1);
+ if (abs_qc > NUM_BASE_LEVELS) {
+ const int br_ctx = get_br_ctx_simple(levels, ci, bhl, tx_class);
+ rate += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]);
+ }
return rate;
}
+
+ int rate_low = 0;
+ const int rate = get_two_coeff_cost_simple(
+ ci, abs_qc, coeff_ctx, txb_costs, bhl, tx_class, levels, &rate_low);
+
+ // update_coeff_simple_nz is only invoked for AC coefficients (si > 0), so
+ // dequant_ac (dequant[1]) can be used directly without a (ci > 0) branch.
+ int dqv = ctx->dequant_ac;
+ if (ctx->iqmatrix != NULL) {
+ dqv =
+ ((ctx->iqmatrix[ci] * dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
+ }
+ const tran_low_t abs_qc_low = abs_qc - 1;
+ const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift;
+ int64_t dist_diff_0, dist_diff_low_0;
+ if (qmatrix == NULL) {
+ const int64_t abs_tqc2 = (int64_t)abs_tqc << 1;
+ dist_diff_0 =
+ ((int64_t)abs_dqc * (abs_dqc - abs_tqc2)) * (1LL << (2 * shift));
+ dist_diff_low_0 = ((int64_t)abs_dqc_low * (abs_dqc_low - abs_tqc2)) *
+ (1LL << (2 * shift));
+ } else {
+ const int64_t qm_val = qmatrix[ci];
+ const int64_t dist0 = get_coeff_dist_qm(abs_tqc, 0, shift, qm_val);
+ dist_diff_0 = get_coeff_dist_qm(abs_tqc, abs_dqc, shift, qm_val) - dist0;
+ dist_diff_low_0 =
+ get_coeff_dist_qm(abs_tqc, abs_dqc_low, shift, qm_val) - dist0;
+ }
+ const int64_t rd = RDCOST(rdmult, rate, dist_diff_0);
+ const int64_t rd_low = RDCOST(rdmult, rate_low, dist_diff_low_0);
+ if (rd_low < rd) {
+ const int sign = (qc < 0) ? 1 : 0;
+ ctx->qcoeff[ci] = (-sign ^ abs_qc_low) + sign;
+ ctx->dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign;
+ levels[get_padded_idx(ci, bhl)] = AOMMIN(abs_qc_low, INT8_MAX);
+ return rate_low;
+ }
+ return rate;
}
}
@@ -211,9 +221,9 @@ static inline void update_skip(int *accu_rate, int64_t accu_dist, int *eob,
}
// TODO(angiebird): use this function whenever it's possible
-static int get_tx_type_cost(const MACROBLOCK *x, const MACROBLOCKD *xd,
- int plane, TX_SIZE tx_size, TX_TYPE tx_type,
- int reduced_tx_set_used) {
+static inline int get_tx_type_cost(const MACROBLOCK *x, const MACROBLOCKD *xd,
+ int plane, TX_SIZE tx_size, TX_TYPE tx_type,
+ int reduced_tx_set_used) {
if (plane > 0) return 0;
const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
@@ -400,16 +410,33 @@ static AOM_FORCE_INLINE void update_coeff_simple_facade(
const tran_low_t *__restrict tcoeff, tran_low_t *__restrict qcoeff,
tran_low_t *__restrict dqcoeff, uint8_t *__restrict levels, int sharpness,
const qm_val_t *iqmatrix, const qm_val_t *qmatrix) {
+ (void)eob;
+ int s = *si;
+ if (s < 1) return;
int local_rate = *accu_rate;
- int local_si = *si;
- for (; local_si >= 1; --local_si) {
- local_rate +=
- update_coeff_simple(local_si, eob, tx_size, tx_class, bhl, rdmult,
- shift, dequant, scan, txb_costs, tcoeff, qcoeff,
- dqcoeff, levels, sharpness, iqmatrix, qmatrix);
+ const TxbSimpleCtx ctx = { rdmult, txb_costs, tcoeff, qcoeff,
+ dqcoeff, levels, iqmatrix, qmatrix,
+ bhl, shift, dequant[1], sharpness };
+ const int (*base_cost)[8] = txb_costs->base_cost;
+ while (s >= 1) {
+ int ci;
+ tran_low_t qc;
+ int coeff_ctx;
+ do {
+ ci = scan[s];
+ assert(ci > 0);
+ qc = qcoeff[ci];
+ coeff_ctx =
+ get_lower_levels_ctx_simple(levels, ci, bhl, tx_size, tx_class);
+ if (qc != 0) break;
+ local_rate += base_cost[coeff_ctx][0];
+ } while (--s >= 1);
+ if (s < 1) break;
+ local_rate += update_coeff_simple_nz(ci, qc, coeff_ctx, tx_class, &ctx);
+ --s;
}
*accu_rate = local_rate;
- *si = local_si;
+ *si = 0;
}
int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
@@ -706,14 +733,14 @@ static AOM_FORCE_INLINE int warehouse_efficients_txb(
for (c = eob - 2; c >= 1; --c) {
const int pos = scan[c];
const int coeff_ctx =
- get_lower_levels_ctx(levels, pos, bhl, tx_size, tx_class);
+ get_lower_levels_ctx_simple(levels, pos, bhl, tx_size, tx_class);
const int level = abs(qcoeff[pos]);
if (level <= NUM_BASE_LEVELS) {
cost +=
base_cost[coeff_ctx][level] + ((level > 0) << AV1_PROB_COST_SHIFT);
} else {
cost += base_cost[coeff_ctx][3] + av1_cost_literal(1);
- const int ctx = get_br_ctx(levels, pos, bhl, tx_class);
+ const int ctx = get_br_ctx_simple(levels, pos, bhl, tx_class);
cost += get_br_cost(level, lps_cost[ctx]);
}
}
@@ -744,13 +771,15 @@ static AOM_FORCE_INLINE int warehouse_efficients_txb(
for (c = eob - 2; c >= 1; --c) {
const int pos = scan[c];
const int coeff_ctx = coeff_contexts[pos];
- const int level = abs(qcoeff[pos]);
- if (level <= NUM_BASE_LEVELS) {
- cost +=
- base_cost[coeff_ctx][level] + ((level > 0) << AV1_PROB_COST_SHIFT);
- } else {
- cost += base_cost[coeff_ctx][3] + av1_cost_literal(1);
- const int ctx = get_br_ctx(levels, pos, bhl, tx_class);
+ const tran_low_t v = qcoeff[pos];
+ if (!v) {
+ cost += base_cost[coeff_ctx][0];
+ continue;
+ }
+ const int level = abs(v);
+ cost += base_cost[coeff_ctx][AOMMIN(level, 3)] + av1_cost_literal(1);
+ if (level > NUM_BASE_LEVELS) {
+ const int ctx = get_br_ctx_simple(levels, pos, bhl, tx_class);
cost += get_br_cost(level, lps_cost[ctx]);
}
}