Commit ccb9ed137f for aom
commit ccb9ed137fbb3ae304d31dc1985d556a707903db
Author: Frank Galligan <fgalligan@google.com>
Date: Fri Oct 30 12:07:42 2026 -0700
av1: Optimize update_coeff_eob and subtract_block
Split non-zero coefficient handling in update_coeff_eob into
update_coeff_eob_nz, bypass update_coeff_simple_nz when sharpness != 0
and |qc| == 1, and add AVX2 paths for 4xN and 8xN blocks in
aom_subtract_block_avx2.
Benchmarking results (incremental vs prev CL / cumulative vs baseline):
- Stock avifenc still-image (Default 1T): -2.47% / -23.31% encode time
- Stock avifenc still-image (Default 8T): -1.82% / -18.73% encode time
- 48-image multi-rate corpus (Full Suite 1T): -2.36% / -23.48% time
- Bit-exact output (1T and 8T)
TAG=agy
CONV=b929a630-a6cd-45a0-979c-1767c30178ee
Change-Id: Ibd672abfd184ff276a54bf180f2850b7876dd400
diff --git a/aom_dsp/x86/subtract_avx2.c b/aom_dsp/x86/subtract_avx2.c
index 4dc96d1e4b..9d6b980f02 100644
--- a/aom_dsp/x86/subtract_avx2.c
+++ b/aom_dsp/x86/subtract_avx2.c
@@ -9,94 +9,142 @@
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
+#include <string.h>
#include "config/aom_dsp_rtcd.h"
-static inline void subtract32_avx2(int16_t *diff_ptr, const uint8_t *src_ptr,
- const uint8_t *pred_ptr) {
- __m256i s = _mm256_lddqu_si256((__m256i *)(src_ptr));
- __m256i p = _mm256_lddqu_si256((__m256i *)(pred_ptr));
- __m256i set_one_minusone = _mm256_set1_epi32((int)0xff01ff01);
- __m256i diff0 = _mm256_unpacklo_epi8(s, p);
- __m256i diff1 = _mm256_unpackhi_epi8(s, p);
- diff0 = _mm256_maddubs_epi16(diff0, set_one_minusone);
- diff1 = _mm256_maddubs_epi16(diff1, set_one_minusone);
- _mm256_store_si256((__m256i *)(diff_ptr),
- _mm256_permute2x128_si256(diff0, diff1, 0x20));
- _mm256_store_si256((__m256i *)(diff_ptr + 16),
- _mm256_permute2x128_si256(diff0, diff1, 0x31));
-}
-
static inline void subtract_block_4xn_avx2(
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
- int32_t j = 0;
- for (; j + 3 < rows; j += 4) {
- const __m128i s = _mm_setr_epi32(
- *(const int32_t *)(src_ptr), *(const int32_t *)(src_ptr + src_stride),
- *(const int32_t *)(src_ptr + src_stride * 2),
- *(const int32_t *)(src_ptr + src_stride * 3));
- const __m128i p =
- _mm_setr_epi32(*(const int32_t *)(pred_ptr),
- *(const int32_t *)(pred_ptr + pred_stride),
- *(const int32_t *)(pred_ptr + pred_stride * 2),
- *(const int32_t *)(pred_ptr + pred_stride * 3));
+ int32_t j = rows;
+ // Process 4 rows of 4 pixels at a time in a 256-bit register.
+ for (; j >= 4; j -= 4) {
+ int32_t s0, s1, s2, s3, p0, p1, p2, p3;
+ memcpy(&s0, src_ptr, 4);
+ memcpy(&s1, src_ptr + src_stride, 4);
+ memcpy(&s2, src_ptr + 2 * src_stride, 4);
+ memcpy(&s3, src_ptr + 3 * src_stride, 4);
+ memcpy(&p0, pred_ptr, 4);
+ memcpy(&p1, pred_ptr + pred_stride, 4);
+ memcpy(&p2, pred_ptr + 2 * pred_stride, 4);
+ memcpy(&p3, pred_ptr + 3 * pred_stride, 4);
+
+ const __m128i s = _mm_set_epi32(s3, s2, s1, s0);
+ const __m128i p = _mm_set_epi32(p3, p2, p1, p0);
const __m256i d =
_mm256_sub_epi16(_mm256_cvtepu8_epi16(s), _mm256_cvtepu8_epi16(p));
+
const __m128i d_lo = _mm256_castsi256_si128(d);
const __m128i d_hi = _mm256_extracti128_si256(d, 1);
- _mm_storel_epi64((__m128i *)(diff_ptr), d_lo);
+ _mm_storel_epi64((__m128i *)diff_ptr, d_lo);
_mm_storeh_pi((__m64 *)(diff_ptr + diff_stride), _mm_castsi128_ps(d_lo));
- _mm_storel_epi64((__m128i *)(diff_ptr + diff_stride * 2), d_hi);
- _mm_storeh_pi((__m64 *)(diff_ptr + diff_stride * 3),
+ _mm_storel_epi64((__m128i *)(diff_ptr + 2 * diff_stride), d_hi);
+ _mm_storeh_pi((__m64 *)(diff_ptr + 3 * diff_stride),
_mm_castsi128_ps(d_hi));
- src_ptr += src_stride * 4;
- pred_ptr += pred_stride * 4;
- diff_ptr += diff_stride * 4;
+
+ src_ptr += 4 * src_stride;
+ pred_ptr += 4 * pred_stride;
+ diff_ptr += 4 * diff_stride;
}
- for (; j < rows; j += 2) {
- const __m128i s =
- _mm_setr_epi32(*(const int32_t *)(src_ptr),
- *(const int32_t *)(src_ptr + src_stride), 0, 0);
- const __m128i p =
- _mm_setr_epi32(*(const int32_t *)(pred_ptr),
- *(const int32_t *)(pred_ptr + pred_stride), 0, 0);
+
+ for (; j > 0; j -= 2) {
+ int32_t s0, s1, p0, p1;
+ memcpy(&s0, src_ptr, 4);
+ memcpy(&s1, src_ptr + src_stride, 4);
+ memcpy(&p0, pred_ptr, 4);
+ memcpy(&p1, pred_ptr + pred_stride, 4);
+
+ const __m128i s = _mm_set_epi32(0, 0, s1, s0);
+ const __m128i p = _mm_set_epi32(0, 0, p1, p0);
const __m128i d = _mm_sub_epi16(_mm_cvtepu8_epi16(s), _mm_cvtepu8_epi16(p));
- _mm_storel_epi64((__m128i *)(diff_ptr), d);
+
+ _mm_storel_epi64((__m128i *)diff_ptr, d);
_mm_storeh_pi((__m64 *)(diff_ptr + diff_stride), _mm_castsi128_ps(d));
- src_ptr += src_stride * 2;
- pred_ptr += pred_stride * 2;
- diff_ptr += diff_stride * 2;
+ src_ptr += 2 * src_stride;
+ pred_ptr += 2 * pred_stride;
+ diff_ptr += 2 * diff_stride;
}
}
static inline void subtract_block_8xn_avx2(
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
- for (int32_t j = 0; j < rows; j += 2) {
- const __m128i s = _mm_unpacklo_epi64(
- _mm_loadl_epi64((const __m128i *)(src_ptr)),
+ int32_t j = rows;
+ // Process 4 rows of 8 pixels at a time across two 256-bit registers.
+ for (; j >= 4; j -= 4) {
+ const __m128i s01 = _mm_unpacklo_epi64(
+ _mm_loadl_epi64((const __m128i *)src_ptr),
_mm_loadl_epi64((const __m128i *)(src_ptr + src_stride)));
- const __m128i p = _mm_unpacklo_epi64(
- _mm_loadl_epi64((const __m128i *)(pred_ptr)),
+ const __m128i p01 = _mm_unpacklo_epi64(
+ _mm_loadl_epi64((const __m128i *)pred_ptr),
_mm_loadl_epi64((const __m128i *)(pred_ptr + pred_stride)));
- const __m256i d =
- _mm256_sub_epi16(_mm256_cvtepu8_epi16(s), _mm256_cvtepu8_epi16(p));
- _mm_store_si128((__m128i *)(diff_ptr), _mm256_castsi256_si128(d));
- _mm_store_si128((__m128i *)(diff_ptr + diff_stride),
- _mm256_extracti128_si256(d, 1));
- src_ptr += src_stride * 2;
- pred_ptr += pred_stride * 2;
- diff_ptr += diff_stride * 2;
+ const __m128i s23 = _mm_unpacklo_epi64(
+ _mm_loadl_epi64((const __m128i *)(src_ptr + 2 * src_stride)),
+ _mm_loadl_epi64((const __m128i *)(src_ptr + 3 * src_stride)));
+ const __m128i p23 = _mm_unpacklo_epi64(
+ _mm_loadl_epi64((const __m128i *)(pred_ptr + 2 * pred_stride)),
+ _mm_loadl_epi64((const __m128i *)(pred_ptr + 3 * pred_stride)));
+
+ const __m256i d01 =
+ _mm256_sub_epi16(_mm256_cvtepu8_epi16(s01), _mm256_cvtepu8_epi16(p01));
+ const __m256i d23 =
+ _mm256_sub_epi16(_mm256_cvtepu8_epi16(s23), _mm256_cvtepu8_epi16(p23));
+
+ _mm_storeu_si128((__m128i *)diff_ptr, _mm256_castsi256_si128(d01));
+ _mm_storeu_si128((__m128i *)(diff_ptr + diff_stride),
+ _mm256_extracti128_si256(d01, 1));
+ _mm_storeu_si128((__m128i *)(diff_ptr + 2 * diff_stride),
+ _mm256_castsi256_si128(d23));
+ _mm_storeu_si128((__m128i *)(diff_ptr + 3 * diff_stride),
+ _mm256_extracti128_si256(d23, 1));
+
+ src_ptr += 4 * src_stride;
+ pred_ptr += 4 * pred_stride;
+ diff_ptr += 4 * diff_stride;
}
+
+ for (; j > 0; j -= 2) {
+ const __m128i s01 = _mm_unpacklo_epi64(
+ _mm_loadl_epi64((const __m128i *)src_ptr),
+ _mm_loadl_epi64((const __m128i *)(src_ptr + src_stride)));
+ const __m128i p01 = _mm_unpacklo_epi64(
+ _mm_loadl_epi64((const __m128i *)pred_ptr),
+ _mm_loadl_epi64((const __m128i *)(pred_ptr + pred_stride)));
+
+ const __m256i d01 =
+ _mm256_sub_epi16(_mm256_cvtepu8_epi16(s01), _mm256_cvtepu8_epi16(p01));
+
+ _mm_storeu_si128((__m128i *)diff_ptr, _mm256_castsi256_si128(d01));
+ _mm_storeu_si128((__m128i *)(diff_ptr + diff_stride),
+ _mm256_extracti128_si256(d01, 1));
+ src_ptr += 2 * src_stride;
+ pred_ptr += 2 * pred_stride;
+ diff_ptr += 2 * diff_stride;
+ }
+}
+
+static inline void subtract32_avx2(int16_t *diff_ptr, const uint8_t *src_ptr,
+ const uint8_t *pred_ptr) {
+ __m256i s = _mm256_lddqu_si256((__m256i *)(src_ptr));
+ __m256i p = _mm256_lddqu_si256((__m256i *)(pred_ptr));
+ __m256i set_one_minusone = _mm256_set1_epi32((int)0xff01ff01);
+ __m256i diff0 = _mm256_unpacklo_epi8(s, p);
+ __m256i diff1 = _mm256_unpackhi_epi8(s, p);
+ diff0 = _mm256_maddubs_epi16(diff0, set_one_minusone);
+ diff1 = _mm256_maddubs_epi16(diff1, set_one_minusone);
+ _mm256_store_si256((__m256i *)(diff_ptr),
+ _mm256_permute2x128_si256(diff0, diff1, 0x20));
+ _mm256_store_si256((__m256i *)(diff_ptr + 16),
+ _mm256_permute2x128_si256(diff0, diff1, 0x31));
}
static inline void subtract_block_16xn_avx2(
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
- for (int32_t j = 0; j < rows; j += 2) {
- const __m128i s0 = _mm_lddqu_si128((const __m128i *)(src_ptr));
- const __m128i p0 = _mm_lddqu_si128((const __m128i *)(pred_ptr));
+ int32_t j = rows;
+ for (; j >= 2; j -= 2) {
+ const __m128i s0 = _mm_lddqu_si128((const __m128i *)src_ptr);
+ const __m128i p0 = _mm_lddqu_si128((const __m128i *)pred_ptr);
const __m128i s1 = _mm_lddqu_si128((const __m128i *)(src_ptr + src_stride));
const __m128i p1 =
_mm_lddqu_si128((const __m128i *)(pred_ptr + pred_stride));
@@ -104,11 +152,21 @@ static inline void subtract_block_16xn_avx2(
_mm256_sub_epi16(_mm256_cvtepu8_epi16(s0), _mm256_cvtepu8_epi16(p0));
const __m256i d1 =
_mm256_sub_epi16(_mm256_cvtepu8_epi16(s1), _mm256_cvtepu8_epi16(p1));
- _mm256_store_si256((__m256i *)(diff_ptr), d0);
+ _mm256_store_si256((__m256i *)diff_ptr, d0);
_mm256_store_si256((__m256i *)(diff_ptr + diff_stride), d1);
- src_ptr += src_stride * 2;
- pred_ptr += pred_stride * 2;
- diff_ptr += diff_stride * 2;
+ src_ptr += 2 * src_stride;
+ pred_ptr += 2 * pred_stride;
+ diff_ptr += 2 * diff_stride;
+ }
+ for (; j > 0; --j) {
+ const __m128i s = _mm_lddqu_si128((const __m128i *)src_ptr);
+ const __m128i p = _mm_lddqu_si128((const __m128i *)pred_ptr);
+ const __m256i d =
+ _mm256_sub_epi16(_mm256_cvtepu8_epi16(s), _mm256_cvtepu8_epi16(p));
+ _mm256_store_si256((__m256i *)diff_ptr, d);
+ src_ptr += src_stride;
+ pred_ptr += pred_stride;
+ diff_ptr += diff_stride;
}
}
diff --git a/av1/encoder/txb_rdopt.c b/av1/encoder/txb_rdopt.c
index ed7c059398..89999f72e2 100644
--- a/av1/encoder/txb_rdopt.c
+++ b/av1/encoder/txb_rdopt.c
@@ -104,6 +104,32 @@ typedef struct {
} TxbSimpleCtx;
/*!\endcond */
+/*!\cond */
+typedef struct {
+ int64_t rdmult;
+ int64_t accu_dist;
+ const LV_MAP_EOB_COST *txb_eob_costs;
+ const LV_MAP_COEFF_COST *txb_costs;
+ const int16_t *dequant;
+ const tran_low_t *tcoeff;
+ tran_low_t *qcoeff;
+ tran_low_t *dqcoeff;
+ uint8_t *levels;
+ const qm_val_t *iqmatrix;
+ const qm_val_t *qmatrix;
+ int *nz_ci;
+ int eob;
+ int nz_num;
+ int accu_rate;
+ int bhl;
+ int width;
+ int dc_sign_ctx;
+ int shift;
+ int sharpness;
+ int is_noise_pattern;
+} TxbEobCtx;
+/*!\endcond */
+
#if defined(__GNUC__) || defined(__clang__)
__attribute__((noinline))
#endif
@@ -111,94 +137,245 @@ static int update_coeff_simple_nz(int ci, tran_low_t qc, int coeff_ctx,
TX_CLASS tx_class, const TxbSimpleCtx *ctx) {
assert(ci > 0);
const LV_MAP_COEFF_COST *txb_costs = ctx->txb_costs;
- const int bhl = ctx->bhl;
- const int64_t rdmult = ctx->rdmult;
- const int shift = ctx->shift;
- const int sharpness = ctx->sharpness;
- const qm_val_t *qmatrix = ctx->qmatrix;
- uint8_t *levels = ctx->levels;
const tran_low_t abs_qc = abs(qc);
const tran_low_t abs_tqc = abs(ctx->tcoeff[ci]);
const tran_low_t abs_dqc = abs(ctx->dqcoeff[ci]);
if (abs_qc == 1) {
const int *base_cost = txb_costs->base_cost[coeff_ctx];
const int rate = base_cost[1] + av1_cost_literal(1);
+ if (ctx->sharpness || abs_dqc < abs_tqc) {
+ return rate;
+ }
+
+ const qm_val_t *qmatrix = ctx->qmatrix;
+ const int shift = ctx->shift;
+ int64_t dist_diff_0;
+ if (qmatrix == NULL) {
+ dist_diff_0 = ((int64_t)abs_dqc * (abs_dqc - ((int64_t)abs_tqc * 2))) *
+ (1LL << (2 * shift));
+ } else {
+ const int64_t qm_val = qmatrix[ci];
+ const int64_t dist0 = get_coeff_dist_qm(abs_tqc, 0, shift, qm_val);
+ dist_diff_0 = get_coeff_dist_qm(abs_tqc, abs_dqc, shift, qm_val) - dist0;
+ }
+ const int rate_low = rate - base_cost[5];
+ const int64_t rdmult = ctx->rdmult;
+ const int64_t rd = RDCOST(rdmult, rate, dist_diff_0);
+ const int64_t rd_low = RDCOST(rdmult, rate_low, 0);
+ if (rd_low < rd) {
+ ctx->qcoeff[ci] = 0;
+ ctx->dqcoeff[ci] = 0;
+ ctx->levels[get_padded_idx(ci, ctx->bhl)] = 0;
+ return rate_low;
+ }
+ return rate;
+ } else {
if (abs_dqc < abs_tqc) {
+ int rate = txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)] +
+ av1_cost_literal(1);
+ if (abs_qc > NUM_BASE_LEVELS) {
+ const int br_ctx =
+ get_br_ctx_simple(ctx->levels, ci, ctx->bhl, tx_class);
+ rate += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]);
+ }
return rate;
}
- const int allow_lower_qc = sharpness ? 0 : 1;
+ const int bhl = ctx->bhl;
+ uint8_t *levels = ctx->levels;
+ int rate_low = 0;
+ const int rate = get_two_coeff_cost_simple(
+ ci, abs_qc, coeff_ctx, txb_costs, bhl, tx_class, levels, &rate_low);
+
+ const int allow_lower_qc = ctx->sharpness ? (abs_qc > 1) : 1;
if (allow_lower_qc) {
- int64_t dist_diff_0;
+ // update_coeff_simple_nz is only invoked for AC coefficients (si > 0), so
+ // dequant_ac (dequant[1]) can be used directly without a (ci > 0) branch.
+ int dqv = ctx->dequant_ac;
+ if (ctx->iqmatrix != NULL) {
+ dqv = ((ctx->iqmatrix[ci] * dqv) + (1 << (AOM_QM_BITS - 1))) >>
+ AOM_QM_BITS;
+ }
+ const int shift = ctx->shift;
+ const tran_low_t abs_qc_low = abs_qc - 1;
+ const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift;
+ const qm_val_t *qmatrix = ctx->qmatrix;
+ int64_t dist_diff_0, dist_diff_low_0;
if (qmatrix == NULL) {
- dist_diff_0 = ((int64_t)abs_dqc * (abs_dqc - ((int64_t)abs_tqc * 2))) *
- (1LL << (2 * shift));
+ const int64_t abs_tqc2 = (int64_t)abs_tqc << 1;
+ dist_diff_0 =
+ ((int64_t)abs_dqc * (abs_dqc - abs_tqc2)) * (1LL << (2 * shift));
+ dist_diff_low_0 = ((int64_t)abs_dqc_low * (abs_dqc_low - abs_tqc2)) *
+ (1LL << (2 * shift));
} else {
const int64_t qm_val = qmatrix[ci];
const int64_t dist0 = get_coeff_dist_qm(abs_tqc, 0, shift, qm_val);
dist_diff_0 =
get_coeff_dist_qm(abs_tqc, abs_dqc, shift, qm_val) - dist0;
+ dist_diff_low_0 =
+ get_coeff_dist_qm(abs_tqc, abs_dqc_low, shift, qm_val) - dist0;
}
- const int rate_low = rate - base_cost[5];
+ const int64_t rdmult = ctx->rdmult;
const int64_t rd = RDCOST(rdmult, rate, dist_diff_0);
- const int64_t rd_low = RDCOST(rdmult, rate_low, 0);
+ const int64_t rd_low = RDCOST(rdmult, rate_low, dist_diff_low_0);
if (rd_low < rd) {
- ctx->qcoeff[ci] = 0;
- ctx->dqcoeff[ci] = 0;
- levels[get_padded_idx(ci, bhl)] = 0;
+ const int sign = (qc < 0) ? 1 : 0;
+ ctx->qcoeff[ci] = (-sign ^ abs_qc_low) + sign;
+ ctx->dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign;
+ levels[get_padded_idx(ci, bhl)] = AOMMIN(abs_qc_low, INT8_MAX);
return rate_low;
}
}
return rate;
+ }
+}
+
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
+static int update_coeff_eob_nz(int si, int ci, tran_low_t qc, int coeff_ctx,
+ TX_CLASS tx_class, TxbEobCtx *ctx) {
+ const LV_MAP_COEFF_COST *txb_costs = ctx->txb_costs;
+ const int bhl = ctx->bhl;
+ const int dc_sign_ctx = ctx->dc_sign_ctx;
+ uint8_t *levels = ctx->levels;
+ const tran_low_t abs_qc = abs(qc);
+ const tran_low_t tqc = ctx->tcoeff[ci];
+ const tran_low_t dqc = ctx->dqcoeff[ci];
+ const int sign = (qc < 0) ? 1 : 0;
+ const int shift = ctx->shift;
+ const qm_val_t *qmatrix = ctx->qmatrix;
+
+ const int new_eob = si + 1;
+ const int min_eob_cutoff = ctx->is_noise_pattern ? 8 : 5;
+ const int qc_threshold = (si <= min_eob_cutoff) ? 2 : 1;
+ const int sharpness = ctx->sharpness;
+ const int allow_lower_qc = sharpness ? (abs_qc > qc_threshold) : 1;
+ const int allow_new_eob = (sharpness == 0 || new_eob >= min_eob_cutoff);
+
+ int rate = get_coeff_cost_general(0, ci, abs_qc, sign, coeff_ctx, dc_sign_ctx,
+ txb_costs, bhl, tx_class, levels);
+ int64_t dist;
+ int64_t dist0 = 0;
+ int64_t qm_val = 0;
+ if (qmatrix == NULL) {
+ const int64_t tqc2 = (int64_t)tqc * 2;
+ dist = ((int64_t)dqc * (dqc - tqc2)) * (1LL << (2 * shift));
} else {
- if (abs_dqc < abs_tqc) {
- int rate = txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)] +
- av1_cost_literal(1);
- if (abs_qc > NUM_BASE_LEVELS) {
- const int br_ctx = get_br_ctx_simple(levels, ci, bhl, tx_class);
- rate += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]);
- }
- return rate;
- }
+ qm_val = qmatrix[ci];
+ dist0 = get_coeff_dist_qm(tqc, 0, shift, qm_val);
+ dist = get_coeff_dist_qm(tqc, dqc, shift, qm_val) - dist0;
+ }
- int rate_low = 0;
- const int rate = get_two_coeff_cost_simple(
- ci, abs_qc, coeff_ctx, txb_costs, bhl, tx_class, levels, &rate_low);
+ if (!allow_lower_qc && !allow_new_eob) {
+ ctx->accu_dist += dist;
+ ctx->nz_ci[ctx->nz_num] = ci;
+ ++ctx->nz_num;
+ return ctx->accu_rate + rate;
+ }
- // update_coeff_simple_nz is only invoked for AC coefficients (si > 0), so
- // dequant_ac (dequant[1]) can be used directly without a (ci > 0) branch.
- int dqv = ctx->dequant_ac;
- if (ctx->iqmatrix != NULL) {
- dqv =
- ((ctx->iqmatrix[ci] * dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
- }
- const tran_low_t abs_qc_low = abs_qc - 1;
- const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift;
- int64_t dist_diff_0, dist_diff_low_0;
+ const int64_t dist_orig = dist;
+ tran_low_t qc_low = 0, dqc_low = 0;
+ tran_low_t abs_qc_low = 0;
+ int64_t dist_low = 0;
+ if (abs_qc > 1) {
+ const int dqv = get_dqv(ctx->dequant, ci, ctx->iqmatrix);
+ get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low);
+ abs_qc_low = abs_qc - 1;
if (qmatrix == NULL) {
- const int64_t abs_tqc2 = (int64_t)abs_tqc << 1;
- dist_diff_0 =
- ((int64_t)abs_dqc * (abs_dqc - abs_tqc2)) * (1LL << (2 * shift));
- dist_diff_low_0 = ((int64_t)abs_dqc_low * (abs_dqc_low - abs_tqc2)) *
- (1LL << (2 * shift));
+ const int64_t tqc2 = (int64_t)tqc * 2;
+ dist_low = ((int64_t)dqc_low * (dqc_low - tqc2)) * (1LL << (2 * shift));
} else {
- const int64_t qm_val = qmatrix[ci];
- const int64_t dist0 = get_coeff_dist_qm(abs_tqc, 0, shift, qm_val);
- dist_diff_0 = get_coeff_dist_qm(abs_tqc, abs_dqc, shift, qm_val) - dist0;
- dist_diff_low_0 =
- get_coeff_dist_qm(abs_tqc, abs_dqc_low, shift, qm_val) - dist0;
+ dist_low = get_coeff_dist_qm(tqc, dqc_low, shift, qm_val) - dist0;
}
- const int64_t rd = RDCOST(rdmult, rate, dist_diff_0);
- const int64_t rd_low = RDCOST(rdmult, rate_low, dist_diff_low_0);
+ }
+
+ const int64_t rdmult = ctx->rdmult;
+ int accu_rate = ctx->accu_rate;
+ int64_t accu_dist = ctx->accu_dist;
+ int64_t rd = RDCOST(rdmult, accu_rate + rate, accu_dist + dist);
+ int lower_level = 0;
+
+ if (allow_lower_qc) {
+ const int rate_low =
+ (abs_qc == 1) ? txb_costs->base_cost[coeff_ctx][0]
+ : get_coeff_cost_general(
+ 0, ci, abs_qc_low, sign, coeff_ctx, dc_sign_ctx,
+ txb_costs, bhl, tx_class, levels);
+ const int64_t rd_low =
+ RDCOST(rdmult, accu_rate + rate_low, accu_dist + dist_low);
if (rd_low < rd) {
- const int sign = (qc < 0) ? 1 : 0;
- ctx->qcoeff[ci] = (-sign ^ abs_qc_low) + sign;
- ctx->dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign;
- levels[get_padded_idx(ci, bhl)] = AOMMIN(abs_qc_low, INT8_MAX);
- return rate_low;
+ lower_level = 1;
+ rd = rd_low;
+ rate = rate_low;
+ dist = dist_low;
}
- return rate;
}
+
+ if (allow_new_eob) {
+ int lower_level_new_eob = 0;
+ const int coeff_ctx_new_eob = get_lower_levels_ctx_eob(bhl, ctx->width, si);
+ const int new_eob_cost =
+ get_eob_cost(new_eob, ctx->txb_eob_costs, txb_costs, tx_class);
+ int rate_coeff_eob =
+ new_eob_cost + get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx_new_eob,
+ dc_sign_ctx, txb_costs, bhl,
+ tx_class);
+ int64_t dist_new_eob = dist_orig;
+ int64_t rd_new_eob = RDCOST(rdmult, rate_coeff_eob, dist_new_eob);
+
+ if (abs_qc_low > 0) {
+ const int rate_coeff_eob_low =
+ new_eob_cost + get_coeff_cost_eob(ci, abs_qc_low, sign,
+ coeff_ctx_new_eob, dc_sign_ctx,
+ txb_costs, bhl, tx_class);
+ const int64_t dist_new_eob_low = dist_low;
+ const int64_t rd_new_eob_low =
+ RDCOST(rdmult, rate_coeff_eob_low, dist_new_eob_low);
+ if (rd_new_eob_low < rd_new_eob) {
+ lower_level_new_eob = 1;
+ rd_new_eob = rd_new_eob_low;
+ rate_coeff_eob = rate_coeff_eob_low;
+ dist_new_eob = dist_new_eob_low;
+ }
+ }
+
+ if (rd_new_eob < rd) {
+ tran_low_t *qcoeff = ctx->qcoeff;
+ tran_low_t *dqcoeff = ctx->dqcoeff;
+ const int *nz_ci = ctx->nz_ci;
+ const int nz_num = ctx->nz_num;
+ for (int ni = 0; ni < nz_num; ++ni) {
+ int last_ci = nz_ci[ni];
+ levels[get_padded_idx(last_ci, bhl)] = 0;
+ qcoeff[last_ci] = 0;
+ dqcoeff[last_ci] = 0;
+ }
+ ctx->eob = new_eob;
+ ctx->nz_num = 0;
+ accu_rate = rate_coeff_eob;
+ ctx->accu_dist = dist_new_eob;
+ lower_level = lower_level_new_eob;
+ } else {
+ accu_rate += rate;
+ ctx->accu_dist = accu_dist + dist;
+ }
+ } else {
+ accu_rate += rate;
+ ctx->accu_dist = accu_dist + dist;
+ }
+
+ tran_low_t *qcoeff = ctx->qcoeff;
+ if (lower_level) {
+ qcoeff[ci] = qc_low;
+ ctx->dqcoeff[ci] = dqc_low;
+ levels[get_padded_idx(ci, bhl)] = AOMMIN(abs_qc_low, INT8_MAX);
+ }
+ if (qcoeff[ci]) {
+ ctx->nz_ci[ctx->nz_num] = ci;
+ ++ctx->nz_num;
+ }
+ return accu_rate;
}
static inline void update_skip(int *accu_rate, int64_t accu_dist, int *eob,
@@ -266,140 +443,37 @@ static AOM_FORCE_INLINE void update_coeff_eob_facade(
tran_low_t *__restrict dqcoeff, uint8_t *__restrict levels, int sharpness,
const qm_val_t *iqmatrix, const qm_val_t *qmatrix, int max_nz_num,
int is_noise_pattern) {
+ int s = *si;
+ if (s < 0) return;
int local_rate = *accu_rate;
- int64_t local_dist = *accu_dist;
- int local_eob = *eob;
- int local_nz_num = *nz_num;
- int local_si = *si;
- for (; local_si >= 0 && local_nz_num <= max_nz_num; --local_si) {
- assert(local_si != local_eob - 1);
- const int ci = scan[local_si];
- const tran_low_t qc = qcoeff[ci];
- const int coeff_ctx =
- get_lower_levels_ctx(levels, ci, bhl, tx_size, tx_class);
- if (qc == 0) {
- local_rate += txb_costs->base_cost[coeff_ctx][0];
- } else {
- int lower_level = 0;
- const tran_low_t abs_qc = abs(qc);
- const tran_low_t tqc = tcoeff[ci];
- const tran_low_t dqc = dqcoeff[ci];
- const int sign = (qc < 0) ? 1 : 0;
-
- tran_low_t qc_low = 0, dqc_low = 0;
- tran_low_t abs_qc_low = 0;
- int rate_low;
-
- if (abs_qc == 1) {
- rate_low = txb_costs->base_cost[coeff_ctx][0];
- } else {
- const int dqv = get_dqv(dequant, ci, iqmatrix);
- get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low);
- abs_qc_low = abs_qc - 1;
- rate_low = get_coeff_cost_general(0, ci, abs_qc_low, sign, coeff_ctx,
- dc_sign_ctx, txb_costs, bhl, tx_class,
- levels);
- }
-
- int64_t dist, dist_low;
- if (qmatrix == NULL) {
- const int64_t tqc2 = (int64_t)tqc * 2;
- dist = ((int64_t)dqc * (dqc - tqc2)) * (1LL << (2 * shift));
- dist_low = (abs_qc == 1) ? 0
- : (((int64_t)dqc_low * (dqc_low - tqc2)) *
- (1LL << (2 * shift)));
- } else {
- const int64_t qm_val = qmatrix[ci];
- const int64_t dist0 = get_coeff_dist_qm(tqc, 0, shift, qm_val);
- dist = get_coeff_dist_qm(tqc, dqc, shift, qm_val) - dist0;
- dist_low =
- (abs_qc == 1)
- ? 0
- : (get_coeff_dist_qm(tqc, dqc_low, shift, qm_val) - dist0);
- }
-
- int rate =
- get_coeff_cost_general(0, ci, abs_qc, sign, coeff_ctx, dc_sign_ctx,
- txb_costs, bhl, tx_class, levels);
- int64_t rd = RDCOST(rdmult, local_rate + rate, local_dist + dist);
- int64_t rd_low =
- RDCOST(rdmult, local_rate + rate_low, local_dist + dist_low);
-
- int lower_level_new_eob = 0;
- const int new_eob = local_si + 1;
- const int coeff_ctx_new_eob =
- get_lower_levels_ctx_eob(bhl, width, local_si);
- const int new_eob_cost =
- get_eob_cost(new_eob, txb_eob_costs, txb_costs, tx_class);
- int rate_coeff_eob =
- new_eob_cost + get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx_new_eob,
- dc_sign_ctx, txb_costs, bhl,
- tx_class);
- int64_t dist_new_eob = dist;
- int64_t rd_new_eob = RDCOST(rdmult, rate_coeff_eob, dist_new_eob);
-
- if (abs_qc_low > 0) {
- const int rate_coeff_eob_low =
- new_eob_cost + get_coeff_cost_eob(ci, abs_qc_low, sign,
- coeff_ctx_new_eob, dc_sign_ctx,
- txb_costs, bhl, tx_class);
- const int64_t dist_new_eob_low = dist_low;
- const int64_t rd_new_eob_low =
- RDCOST(rdmult, rate_coeff_eob_low, dist_new_eob_low);
- if (rd_new_eob_low < rd_new_eob) {
- lower_level_new_eob = 1;
- rd_new_eob = rd_new_eob_low;
- rate_coeff_eob = rate_coeff_eob_low;
- dist_new_eob = dist_new_eob_low;
- }
- }
-
- const int min_eob_cutoff = is_noise_pattern ? 8 : 5;
- const int qc_threshold = (local_si <= min_eob_cutoff) ? 2 : 1;
- const int allow_lower_qc = sharpness ? abs_qc > qc_threshold : 1;
-
- if (allow_lower_qc) {
- if (rd_low < rd) {
- lower_level = 1;
- rd = rd_low;
- rate = rate_low;
- dist = dist_low;
- }
- }
-
- if ((sharpness == 0 || new_eob >= min_eob_cutoff) && rd_new_eob < rd) {
- for (int ni = 0; ni < local_nz_num; ++ni) {
- int last_ci = nz_ci[ni];
- levels[get_padded_idx(last_ci, bhl)] = 0;
- qcoeff[last_ci] = 0;
- dqcoeff[last_ci] = 0;
- }
- local_eob = new_eob;
- local_nz_num = 0;
- local_rate = rate_coeff_eob;
- local_dist = dist_new_eob;
- lower_level = lower_level_new_eob;
- } else {
- local_rate += rate;
- local_dist += dist;
- }
-
- if (lower_level) {
- qcoeff[ci] = qc_low;
- dqcoeff[ci] = dqc_low;
- levels[get_padded_idx(ci, bhl)] = AOMMIN(abs_qc_low, INT8_MAX);
- }
- if (qcoeff[ci]) {
- nz_ci[local_nz_num] = ci;
- ++local_nz_num;
- }
- }
+ TxbEobCtx ctx = {
+ rdmult, *accu_dist, txb_eob_costs, txb_costs, dequant, tcoeff,
+ qcoeff, dqcoeff, levels, iqmatrix, qmatrix, nz_ci,
+ *eob, *nz_num, local_rate, bhl, width, dc_sign_ctx,
+ shift, sharpness, is_noise_pattern
+ };
+ const int (*base_cost)[8] = txb_costs->base_cost;
+ while (s >= 0 && ctx.nz_num <= max_nz_num) {
+ int ci;
+ tran_low_t qc;
+ int coeff_ctx;
+ do {
+ ci = scan[s];
+ qc = qcoeff[ci];
+ coeff_ctx = get_lower_levels_ctx(levels, ci, bhl, tx_size, tx_class);
+ if (qc != 0) break;
+ local_rate += base_cost[coeff_ctx][0];
+ } while (--s >= 0);
+ if (s < 0) break;
+ ctx.accu_rate = local_rate;
+ local_rate = update_coeff_eob_nz(s, ci, qc, coeff_ctx, tx_class, &ctx);
+ --s;
}
*accu_rate = local_rate;
- *accu_dist = local_dist;
- *eob = local_eob;
- *nz_num = local_nz_num;
- *si = local_si;
+ *accu_dist = ctx.accu_dist;
+ *eob = ctx.eob;
+ *nz_num = ctx.nz_num;
+ *si = s;
}
static AOM_FORCE_INLINE void update_coeff_simple_facade(
@@ -432,6 +506,11 @@ static AOM_FORCE_INLINE void update_coeff_simple_facade(
local_rate += base_cost[coeff_ctx][0];
} while (--s >= 1);
if (s < 1) break;
+ if (sharpness != 0 && (unsigned int)(qc + 1) <= 2) {
+ local_rate += base_cost[coeff_ctx][1] + av1_cost_literal(1);
+ --s;
+ continue;
+ }
local_rate += update_coeff_simple_nz(ci, qc, coeff_ctx, tx_class, &ctx);
--s;
}