Commit c5e59d1982 for aom
commit c5e59d1982aea952ad4178f5b3e67f90645d0203
Author: Frank Galligan <fgalligan@google.com>
Date: Fri Oct 30 12:07:44 2026 -0700
av1: Optimize subtract_block and lower_levels_ctx
Streamline 2-row loops in subtract_block_4xn_avx2 and
subtract_block_8xn_avx2 using xx_loadl_32 and _mm_loadl_epi64, hoist
stride and nz_map_ctx_offset into get_lower_levels_ctx_fast, and split
out warehouse_efficients_txb_multi for eob > 1.
Benchmarking results (incremental vs prev CL / cumulative vs baseline):
- Stock avifenc still-image (Default 1T): -0.40% / -23.61% encode time
- Stock avifenc still-image (Default 8T): -0.10% / -18.81% encode time
- 48-image multi-rate corpus (Full Suite 1T): -0.63% / -23.96% time
- Bit-exact output (1T and 8T)
TAG=agy
CONV=b929a630-a6cd-45a0-979c-1767c30178ee
Change-Id: I38c08e66f44915a0da97de0441fb17acfeceb8a7
diff --git a/aom_dsp/x86/subtract_avx2.c b/aom_dsp/x86/subtract_avx2.c
index 9d6b980f02..ff45d0f0c9 100644
--- a/aom_dsp/x86/subtract_avx2.c
+++ b/aom_dsp/x86/subtract_avx2.c
@@ -8,119 +8,73 @@
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
+#include <assert.h>
#include <immintrin.h>
-#include <string.h>
+#include <stdint.h>
#include "config/aom_dsp_rtcd.h"
+#include "aom_dsp/x86/synonyms.h"
static inline void subtract_block_4xn_avx2(
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
- int32_t j = rows;
- // Process 4 rows of 4 pixels at a time in a 256-bit register.
- for (; j >= 4; j -= 4) {
- int32_t s0, s1, s2, s3, p0, p1, p2, p3;
- memcpy(&s0, src_ptr, 4);
- memcpy(&s1, src_ptr + src_stride, 4);
- memcpy(&s2, src_ptr + 2 * src_stride, 4);
- memcpy(&s3, src_ptr + 3 * src_stride, 4);
- memcpy(&p0, pred_ptr, 4);
- memcpy(&p1, pred_ptr + pred_stride, 4);
- memcpy(&p2, pred_ptr + 2 * pred_stride, 4);
- memcpy(&p3, pred_ptr + 3 * pred_stride, 4);
-
- const __m128i s = _mm_set_epi32(s3, s2, s1, s0);
- const __m128i p = _mm_set_epi32(p3, p2, p1, p0);
- const __m256i d =
- _mm256_sub_epi16(_mm256_cvtepu8_epi16(s), _mm256_cvtepu8_epi16(p));
-
- const __m128i d_lo = _mm256_castsi256_si128(d);
- const __m128i d_hi = _mm256_extracti128_si256(d, 1);
- _mm_storel_epi64((__m128i *)diff_ptr, d_lo);
- _mm_storeh_pi((__m64 *)(diff_ptr + diff_stride), _mm_castsi128_ps(d_lo));
- _mm_storel_epi64((__m128i *)(diff_ptr + 2 * diff_stride), d_hi);
- _mm_storeh_pi((__m64 *)(diff_ptr + 3 * diff_stride),
- _mm_castsi128_ps(d_hi));
-
- src_ptr += 4 * src_stride;
- pred_ptr += 4 * pred_stride;
- diff_ptr += 4 * diff_stride;
- }
-
- for (; j > 0; j -= 2) {
- int32_t s0, s1, p0, p1;
- memcpy(&s0, src_ptr, 4);
- memcpy(&s1, src_ptr + src_stride, 4);
- memcpy(&p0, pred_ptr, 4);
- memcpy(&p1, pred_ptr + pred_stride, 4);
+ assert(rows >= 2 && (rows & 1) == 0);
+ int32_t j = rows >> 1;
+ // Process 2 rows of 4 pixels per iteration using 128-bit XMM registers
+ // rather than 4 rows in a 256-bit YMM register:
+ // 1. xx_loadl_32 + _mm_unpacklo_epi32 loads directly from memory into XMM
+ // without GPR-to-XMM inserts (_mm_set_epi32 / vpinsrd on Port 5).
+ // 2. Staying in a single 128-bit lane avoids the cross-lane latency of
+ // _mm256_cvtepu8_epi16 and _mm256_extracti128_si256 needed for 8-byte
+ // per-row stores.
+ // 3. Stepping pointers by stride each row avoids 3 * stride LEA addressing
+ // and eliminates a separate 2-row remainder loop.
+ do {
+ const __m128i s0 = xx_loadl_32(src_ptr);
+ src_ptr += src_stride;
+ const __m128i s1 = xx_loadl_32(src_ptr);
+ src_ptr += src_stride;
+ const __m128i p0 = xx_loadl_32(pred_ptr);
+ pred_ptr += pred_stride;
+ const __m128i p1 = xx_loadl_32(pred_ptr);
+ pred_ptr += pred_stride;
- const __m128i s = _mm_set_epi32(0, 0, s1, s0);
- const __m128i p = _mm_set_epi32(0, 0, p1, p0);
- const __m128i d = _mm_sub_epi16(_mm_cvtepu8_epi16(s), _mm_cvtepu8_epi16(p));
+ const __m128i d =
+ _mm_sub_epi16(_mm_cvtepu8_epi16(_mm_unpacklo_epi32(s0, s1)),
+ _mm_cvtepu8_epi16(_mm_unpacklo_epi32(p0, p1)));
_mm_storel_epi64((__m128i *)diff_ptr, d);
- _mm_storeh_pi((__m64 *)(diff_ptr + diff_stride), _mm_castsi128_ps(d));
- src_ptr += 2 * src_stride;
- pred_ptr += 2 * pred_stride;
- diff_ptr += 2 * diff_stride;
- }
+ diff_ptr += diff_stride;
+ _mm_storeh_pi((__m64 *)diff_ptr, _mm_castsi128_ps(d));
+ diff_ptr += diff_stride;
+ } while (--j != 0);
}
static inline void subtract_block_8xn_avx2(
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
- int32_t j = rows;
- // Process 4 rows of 8 pixels at a time across two 256-bit registers.
- for (; j >= 4; j -= 4) {
- const __m128i s01 = _mm_unpacklo_epi64(
- _mm_loadl_epi64((const __m128i *)src_ptr),
- _mm_loadl_epi64((const __m128i *)(src_ptr + src_stride)));
- const __m128i p01 = _mm_unpacklo_epi64(
- _mm_loadl_epi64((const __m128i *)pred_ptr),
- _mm_loadl_epi64((const __m128i *)(pred_ptr + pred_stride)));
- const __m128i s23 = _mm_unpacklo_epi64(
- _mm_loadl_epi64((const __m128i *)(src_ptr + 2 * src_stride)),
- _mm_loadl_epi64((const __m128i *)(src_ptr + 3 * src_stride)));
- const __m128i p23 = _mm_unpacklo_epi64(
- _mm_loadl_epi64((const __m128i *)(pred_ptr + 2 * pred_stride)),
- _mm_loadl_epi64((const __m128i *)(pred_ptr + 3 * pred_stride)));
-
- const __m256i d01 =
- _mm256_sub_epi16(_mm256_cvtepu8_epi16(s01), _mm256_cvtepu8_epi16(p01));
- const __m256i d23 =
- _mm256_sub_epi16(_mm256_cvtepu8_epi16(s23), _mm256_cvtepu8_epi16(p23));
-
- _mm_storeu_si128((__m128i *)diff_ptr, _mm256_castsi256_si128(d01));
- _mm_storeu_si128((__m128i *)(diff_ptr + diff_stride),
- _mm256_extracti128_si256(d01, 1));
- _mm_storeu_si128((__m128i *)(diff_ptr + 2 * diff_stride),
- _mm256_castsi256_si128(d23));
- _mm_storeu_si128((__m128i *)(diff_ptr + 3 * diff_stride),
- _mm256_extracti128_si256(d23, 1));
-
- src_ptr += 4 * src_stride;
- pred_ptr += 4 * pred_stride;
- diff_ptr += 4 * diff_stride;
- }
-
- for (; j > 0; j -= 2) {
- const __m128i s01 = _mm_unpacklo_epi64(
- _mm_loadl_epi64((const __m128i *)src_ptr),
- _mm_loadl_epi64((const __m128i *)(src_ptr + src_stride)));
- const __m128i p01 = _mm_unpacklo_epi64(
- _mm_loadl_epi64((const __m128i *)pred_ptr),
- _mm_loadl_epi64((const __m128i *)(pred_ptr + pred_stride)));
+ assert(rows >= 2 && (rows & 1) == 0);
+ int32_t j = rows >> 1;
+ // Process 2 rows of 8 pixels per iteration using a 256-bit register.
+ do {
+ const __m128i s0 = _mm_loadl_epi64((const __m128i *)src_ptr);
+ src_ptr += src_stride;
+ const __m128i s1 = _mm_loadl_epi64((const __m128i *)src_ptr);
+ src_ptr += src_stride;
+ const __m128i p0 = _mm_loadl_epi64((const __m128i *)pred_ptr);
+ pred_ptr += pred_stride;
+ const __m128i p1 = _mm_loadl_epi64((const __m128i *)pred_ptr);
+ pred_ptr += pred_stride;
const __m256i d01 =
- _mm256_sub_epi16(_mm256_cvtepu8_epi16(s01), _mm256_cvtepu8_epi16(p01));
+ _mm256_sub_epi16(_mm256_cvtepu8_epi16(_mm_unpacklo_epi64(s0, s1)),
+ _mm256_cvtepu8_epi16(_mm_unpacklo_epi64(p0, p1)));
_mm_storeu_si128((__m128i *)diff_ptr, _mm256_castsi256_si128(d01));
- _mm_storeu_si128((__m128i *)(diff_ptr + diff_stride),
- _mm256_extracti128_si256(d01, 1));
- src_ptr += 2 * src_stride;
- pred_ptr += 2 * pred_stride;
- diff_ptr += 2 * diff_stride;
- }
+ diff_ptr += diff_stride;
+ _mm_storeu_si128((__m128i *)diff_ptr, _mm256_extracti128_si256(d01, 1));
+ diff_ptr += diff_stride;
+ } while (--j != 0);
}
static inline void subtract32_avx2(int16_t *diff_ptr, const uint8_t *src_ptr,
@@ -141,62 +95,61 @@ static inline void subtract32_avx2(int16_t *diff_ptr, const uint8_t *src_ptr,
static inline void subtract_block_16xn_avx2(
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
- int32_t j = rows;
- for (; j >= 2; j -= 2) {
+ assert(rows >= 2 && (rows & 1) == 0);
+ int32_t j = rows >> 1;
+ do {
const __m128i s0 = _mm_lddqu_si128((const __m128i *)src_ptr);
+ src_ptr += src_stride;
+ const __m128i s1 = _mm_lddqu_si128((const __m128i *)src_ptr);
+ src_ptr += src_stride;
const __m128i p0 = _mm_lddqu_si128((const __m128i *)pred_ptr);
- const __m128i s1 = _mm_lddqu_si128((const __m128i *)(src_ptr + src_stride));
- const __m128i p1 =
- _mm_lddqu_si128((const __m128i *)(pred_ptr + pred_stride));
+ pred_ptr += pred_stride;
+ const __m128i p1 = _mm_lddqu_si128((const __m128i *)pred_ptr);
+ pred_ptr += pred_stride;
const __m256i d0 =
_mm256_sub_epi16(_mm256_cvtepu8_epi16(s0), _mm256_cvtepu8_epi16(p0));
const __m256i d1 =
_mm256_sub_epi16(_mm256_cvtepu8_epi16(s1), _mm256_cvtepu8_epi16(p1));
_mm256_store_si256((__m256i *)diff_ptr, d0);
- _mm256_store_si256((__m256i *)(diff_ptr + diff_stride), d1);
- src_ptr += 2 * src_stride;
- pred_ptr += 2 * pred_stride;
- diff_ptr += 2 * diff_stride;
- }
- for (; j > 0; --j) {
- const __m128i s = _mm_lddqu_si128((const __m128i *)src_ptr);
- const __m128i p = _mm_lddqu_si128((const __m128i *)pred_ptr);
- const __m256i d =
- _mm256_sub_epi16(_mm256_cvtepu8_epi16(s), _mm256_cvtepu8_epi16(p));
- _mm256_store_si256((__m256i *)diff_ptr, d);
- src_ptr += src_stride;
- pred_ptr += pred_stride;
diff_ptr += diff_stride;
- }
+ _mm256_store_si256((__m256i *)diff_ptr, d1);
+ diff_ptr += diff_stride;
+ } while (--j != 0);
}
static inline void subtract_block_32xn_avx2(
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
- for (int32_t j = 0; j < rows; ++j) {
+ assert(rows > 0);
+ int32_t j = rows;
+ do {
subtract32_avx2(diff_ptr, src_ptr, pred_ptr);
src_ptr += src_stride;
pred_ptr += pred_stride;
diff_ptr += diff_stride;
- }
+ } while (--j != 0);
}
static inline void subtract_block_64xn_avx2(
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
- for (int32_t j = 0; j < rows; ++j) {
+ assert(rows > 0);
+ int32_t j = rows;
+ do {
subtract32_avx2(diff_ptr, src_ptr, pred_ptr);
subtract32_avx2(diff_ptr + 32, src_ptr + 32, pred_ptr + 32);
src_ptr += src_stride;
pred_ptr += pred_stride;
diff_ptr += diff_stride;
- }
+ } while (--j != 0);
}
static inline void subtract_block_128xn_avx2(
int rows, int16_t *diff_ptr, ptrdiff_t diff_stride, const uint8_t *src_ptr,
ptrdiff_t src_stride, const uint8_t *pred_ptr, ptrdiff_t pred_stride) {
- for (int32_t j = 0; j < rows; ++j) {
+ assert(rows > 0);
+ int32_t j = rows;
+ do {
subtract32_avx2(diff_ptr, src_ptr, pred_ptr);
subtract32_avx2(diff_ptr + 32, src_ptr + 32, pred_ptr + 32);
subtract32_avx2(diff_ptr + 64, src_ptr + 64, pred_ptr + 64);
@@ -204,13 +157,17 @@ static inline void subtract_block_128xn_avx2(
src_ptr += src_stride;
pred_ptr += pred_stride;
diff_ptr += diff_stride;
- }
+ } while (--j != 0);
}
void aom_subtract_block_avx2(int rows, int cols, int16_t *diff_ptr,
ptrdiff_t diff_stride, const uint8_t *src_ptr,
ptrdiff_t src_stride, const uint8_t *pred_ptr,
ptrdiff_t pred_stride) {
+ if (cols >= 16) {
+ assert(((uintptr_t)diff_ptr & 31) == 0 &&
+ ((diff_stride * sizeof(*diff_ptr)) & 31) == 0);
+ }
switch (cols) {
case 4:
subtract_block_4xn_avx2(rows, diff_ptr, diff_stride, src_ptr, src_stride,
@@ -232,18 +189,10 @@ void aom_subtract_block_avx2(int rows, int cols, int16_t *diff_ptr,
subtract_block_64xn_avx2(rows, diff_ptr, diff_stride, src_ptr, src_stride,
pred_ptr, pred_stride);
break;
- case 128:
+ default:
+ assert(cols == 128);
subtract_block_128xn_avx2(rows, diff_ptr, diff_stride, src_ptr,
src_stride, pred_ptr, pred_stride);
break;
- default:
-#if HAVE_X86_ASM
- aom_subtract_block_sse2(rows, cols, diff_ptr, diff_stride, src_ptr,
- src_stride, pred_ptr, pred_stride);
-#else
- aom_subtract_block_c(rows, cols, diff_ptr, diff_stride, src_ptr,
- src_stride, pred_ptr, pred_stride);
-#endif
- break;
}
}
diff --git a/av1/common/txb_common.h b/av1/common/txb_common.h
index 372d7cd6de..b57bd5da22 100644
--- a/av1/common/txb_common.h
+++ b/av1/common/txb_common.h
@@ -182,10 +182,10 @@ static const uint8_t clip_max3[256] = {
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3
};
-static AOM_FORCE_INLINE int get_nz_mag(const uint8_t *const levels,
- const int bhl, const TX_CLASS tx_class) {
+static AOM_FORCE_INLINE int get_nz_mag_stride(const uint8_t *const levels,
+ const intptr_t stride,
+ const TX_CLASS tx_class) {
int mag;
- const ptrdiff_t stride = (1 << bhl) + TX_PAD_HOR;
const uint8_t *const lvs = levels + stride;
// Note: AOMMIN(level, 3) is useless for decoder since level < 3.
@@ -210,6 +210,12 @@ static AOM_FORCE_INLINE int get_nz_mag(const uint8_t *const levels,
return mag;
}
+static AOM_FORCE_INLINE int get_nz_mag(const uint8_t *const levels,
+ const int bhl, const TX_CLASS tx_class) {
+ const intptr_t stride = (1 << bhl) + TX_PAD_HOR;
+ return get_nz_mag_stride(levels, stride, tx_class);
+}
+
#define NZ_MAP_CTX_0 SIG_COEF_CONTEXTS_2D
#define NZ_MAP_CTX_5 (NZ_MAP_CTX_0 + 5)
#define NZ_MAP_CTX_10 (NZ_MAP_CTX_0 + 10)
@@ -311,43 +317,85 @@ static AOM_FORCE_INLINE int get_lower_levels_ctx(const uint8_t *levels,
return get_nz_map_ctx_from_stats(stats, coeff_idx, bhl, tx_size, tx_class);
}
+static AOM_FORCE_INLINE int get_lower_levels_ctx_fast(
+ const uint8_t *levels, int coeff_idx, int bhl, intptr_t stride,
+ const int8_t *nz_map_ctx_offset, TX_CLASS tx_class) {
+ assert(bhl >= MIN_TX_SIZE_LOG2 && bhl <= MAX_TX_SIZE_LOG2 - 1);
+ assert(stride == ((intptr_t)1 << bhl) + TX_PAD_HOR);
+ const uint8_t *const nb = levels + get_padded_idx(coeff_idx, bhl);
+ const uint8_t *const lvs = nb + stride;
+ if (tx_class == TX_CLASS_2D) {
+ if (coeff_idx == 0) return 0;
+ const int offset = nz_map_ctx_offset[coeff_idx];
+ if ((nb[1] | nb[2] | lvs[0] | lvs[1] | lvs[stride]) == 0) return offset;
+ const int mag = clip_max3[lvs[0]] + clip_max3[nb[1]] + clip_max3[lvs[1]] +
+ clip_max3[lvs[stride]] + clip_max3[nb[2]];
+ return offset + AOMMIN((mag + 1) >> 1, 4);
+ } else if (tx_class == TX_CLASS_VERT) {
+ const int offset =
+ nz_map_ctx_offset_1d[coeff_idx & ((int)stride - (TX_PAD_HOR + 1))];
+ if ((lvs[0] | nb[1] | nb[2] | nb[3] | nb[4]) == 0) return offset;
+ const int mag = clip_max3[lvs[0]] + clip_max3[nb[1]] + clip_max3[nb[2]] +
+ clip_max3[nb[3]] + clip_max3[nb[4]];
+ return offset + AOMMIN((mag + 1) >> 1, 4);
+ } else {
+ assert(tx_class == TX_CLASS_HORIZ);
+ const uint8_t *const lvs2 = lvs + stride;
+ const int offset = nz_map_ctx_offset_1d[coeff_idx >> bhl];
+ if ((lvs[0] | nb[1] | lvs2[0] | lvs2[stride] | lvs2[stride * 2]) == 0)
+ return offset;
+ const int mag = clip_max3[lvs[0]] + clip_max3[nb[1]] + clip_max3[lvs2[0]] +
+ clip_max3[lvs2[stride]] + clip_max3[lvs2[stride * 2]];
+ return offset + AOMMIN((mag + 1) >> 1, 4);
+ }
+}
+
// Specialized variant for AC coefficients (coeff_idx > 0), skipping the
// redundant coeff_idx == 0 checks and dispatching on tx_class once.
-static AOM_FORCE_INLINE int get_lower_levels_ctx_simple(const uint8_t *levels,
- int coeff_idx, int bhl,
- TX_SIZE tx_size,
- TX_CLASS tx_class) {
+static AOM_FORCE_INLINE int get_lower_levels_ctx_simple_fast(
+ const uint8_t *levels, int coeff_idx, int bhl, intptr_t stride,
+ const int8_t *nz_map_ctx_offset, TX_CLASS tx_class) {
assert(coeff_idx > 0);
assert(bhl >= MIN_TX_SIZE_LOG2 && bhl <= MAX_TX_SIZE_LOG2 - 1);
+ assert(stride == ((intptr_t)1 << bhl) + TX_PAD_HOR);
const uint8_t *const nb = levels + get_padded_idx(coeff_idx, bhl);
- const ptrdiff_t stride = (1 << bhl) + TX_PAD_HOR;
+ const uint8_t *const lvs = nb + stride;
if (tx_class == TX_CLASS_2D) {
- const int offset = av1_nz_map_ctx_offset[tx_size][coeff_idx];
- if ((nb[1] | nb[2] | nb[stride] | nb[stride + 1] | nb[2 * stride]) == 0)
- return offset;
- const int mag = clip_max3[nb[stride]] + clip_max3[nb[1]] +
- clip_max3[nb[stride + 1]] + clip_max3[nb[2 * stride]] +
- clip_max3[nb[2]];
+ const int offset = nz_map_ctx_offset[coeff_idx];
+ if ((nb[1] | nb[2] | lvs[0] | lvs[1] | lvs[stride]) == 0) return offset;
+ const int mag = clip_max3[lvs[0]] + clip_max3[nb[1]] + clip_max3[lvs[1]] +
+ clip_max3[lvs[stride]] + clip_max3[nb[2]];
return offset + AOMMIN((mag + 1) >> 1, 4);
} else if (tx_class == TX_CLASS_VERT) {
- const int offset = nz_map_ctx_offset_1d[coeff_idx & ((1 << bhl) - 1)];
- if ((nb[stride] | nb[1] | nb[2] | nb[3] | nb[4]) == 0) return offset;
- const int mag = clip_max3[nb[stride]] + clip_max3[nb[1]] +
- clip_max3[nb[2]] + clip_max3[nb[3]] + clip_max3[nb[4]];
+ const int offset =
+ nz_map_ctx_offset_1d[coeff_idx & ((int)stride - (TX_PAD_HOR + 1))];
+ if ((lvs[0] | nb[1] | nb[2] | nb[3] | nb[4]) == 0) return offset;
+ const int mag = clip_max3[lvs[0]] + clip_max3[nb[1]] + clip_max3[nb[2]] +
+ clip_max3[nb[3]] + clip_max3[nb[4]];
return offset + AOMMIN((mag + 1) >> 1, 4);
} else {
assert(tx_class == TX_CLASS_HORIZ);
+ const uint8_t *const lvs2 = lvs + stride;
const int offset = nz_map_ctx_offset_1d[coeff_idx >> bhl];
- if ((nb[stride] | nb[1] | nb[2 * stride] | nb[3 * stride] |
- nb[4 * stride]) == 0)
+ if ((lvs[0] | nb[1] | lvs2[0] | lvs2[stride] | lvs2[stride * 2]) == 0)
return offset;
- const int mag = clip_max3[nb[stride]] + clip_max3[nb[1]] +
- clip_max3[nb[2 * stride]] + clip_max3[nb[3 * stride]] +
- clip_max3[nb[4 * stride]];
+ const int mag = clip_max3[lvs[0]] + clip_max3[nb[1]] + clip_max3[lvs2[0]] +
+ clip_max3[lvs2[stride]] + clip_max3[lvs2[stride * 2]];
return offset + AOMMIN((mag + 1) >> 1, 4);
}
}
+static AOM_FORCE_INLINE int get_lower_levels_ctx_simple(const uint8_t *levels,
+ int coeff_idx, int bhl,
+ TX_SIZE tx_size,
+ TX_CLASS tx_class) {
+ const intptr_t stride = (1 << bhl) + TX_PAD_HOR;
+ const int8_t *const nz_map_ctx =
+ (tx_class == TX_CLASS_2D) ? av1_nz_map_ctx_offset[tx_size] : NULL;
+ return get_lower_levels_ctx_simple_fast(levels, coeff_idx, bhl, stride,
+ nz_map_ctx, tx_class);
+}
+
static inline int get_lower_levels_ctx_general(int is_last, int scan_idx,
int bhl, int width,
const uint8_t *levels,
diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 07c6a921b5..d64390c417 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -376,7 +376,7 @@ enum {
} UENUM1BYTE(QUANT_FUNC);
#if CONFIG_AV1_HIGHBITDEPTH
-static AV1_QUANT_FACADE
+static const AV1_QUANT_FACADE
quant_func_list[AV1_XFORM_QUANT_TYPES][QUANT_FUNC_TYPES] = {
{ av1_quantize_fp_facade, av1_highbd_quantize_fp_facade },
{ av1_quantize_b_facade, av1_highbd_quantize_b_facade },
@@ -384,7 +384,7 @@ static AV1_QUANT_FACADE
{ NULL, NULL }
};
#else
-static AV1_QUANT_FACADE quant_func_list[AV1_XFORM_QUANT_TYPES] = {
+static const AV1_QUANT_FACADE quant_func_list[AV1_XFORM_QUANT_TYPES] = {
av1_quantize_fp_facade, av1_quantize_b_facade, av1_quantize_dc_facade, NULL
};
#endif
@@ -437,11 +437,22 @@ void av1_quant(MACROBLOCK *x, int plane, int block, TxfmParam *txfm_param,
const int n_coeffs = av1_get_max_eob(txfm_param->tx_size);
if (LIKELY(!x->seg_skip_block)) {
#if CONFIG_AV1_HIGHBITDEPTH
- quant_func_list[qparam->xform_quant_idx][txfm_param->is_hbd](
- coeff, n_coeffs, p, qcoeff, dqcoeff, eob, scan_order, qparam);
+ if (LIKELY(!txfm_param->is_hbd &&
+ qparam->xform_quant_idx == AV1_XFORM_QUANT_FP)) {
+ av1_quantize_fp_facade(coeff, n_coeffs, p, qcoeff, dqcoeff, eob,
+ scan_order, qparam);
+ } else {
+ quant_func_list[qparam->xform_quant_idx][txfm_param->is_hbd](
+ coeff, n_coeffs, p, qcoeff, dqcoeff, eob, scan_order, qparam);
+ }
#else
- quant_func_list[qparam->xform_quant_idx](
- coeff, n_coeffs, p, qcoeff, dqcoeff, eob, scan_order, qparam);
+ if (LIKELY(qparam->xform_quant_idx == AV1_XFORM_QUANT_FP)) {
+ av1_quantize_fp_facade(coeff, n_coeffs, p, qcoeff, dqcoeff, eob,
+ scan_order, qparam);
+ } else {
+ quant_func_list[qparam->xform_quant_idx](
+ coeff, n_coeffs, p, qcoeff, dqcoeff, eob, scan_order, qparam);
+ }
#endif
} else {
av1_quantize_skip(n_coeffs, qcoeff, dqcoeff, eob);
diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c
index a8295a8542..42d528b4f5 100644
--- a/av1/encoder/tx_search.c
+++ b/av1/encoder/tx_search.c
@@ -2191,6 +2191,13 @@ static void search_tx_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
const int use_qmatrix =
av1_use_qmatrix(&cm->quant_params, xd, mbmi->segment_id);
+ const qm_val_t *const seg_qmatrix =
+ use_qmatrix ? xd->plane[plane].seg_qmatrix[mbmi->segment_id][tx_size]
+ : NULL;
+ const qm_val_t *const seg_iqmatrix =
+ use_qmatrix ? xd->plane[plane].seg_iqmatrix[mbmi->segment_id][tx_size]
+ : NULL;
+
int num_tx_candidates = TX_TYPES;
if (allowed_tx_mask != 0 && (allowed_tx_mask & (allowed_tx_mask - 1)) == 0) {
txk_map[0] = get_msb(allowed_tx_mask);
@@ -2205,8 +2212,13 @@ static void search_tx_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
rem_tx_mask &= ~(uint16_t)(1 << tx_type);
txfm_param.tx_type = tx_type;
if (use_qmatrix) {
- av1_setup_qmatrix(&cm->quant_params, xd, plane, tx_size, tx_type,
- &quant_param);
+ if (tx_type < IDTX) {
+ quant_param.qmatrix = seg_qmatrix;
+ quant_param.iqmatrix = seg_iqmatrix;
+ } else {
+ quant_param.qmatrix = NULL;
+ quant_param.iqmatrix = NULL;
+ }
}
if (plane == 0) xd->tx_type_map[tx_type_map_idx] = tx_type;
RD_STATS this_rd_stats;
diff --git a/av1/encoder/txb_rdopt.c b/av1/encoder/txb_rdopt.c
index 89999f72e2..8384ee4a4a 100644
--- a/av1/encoder/txb_rdopt.c
+++ b/av1/encoder/txb_rdopt.c
@@ -141,7 +141,7 @@ static int update_coeff_simple_nz(int ci, tran_low_t qc, int coeff_ctx,
const tran_low_t abs_tqc = abs(ctx->tcoeff[ci]);
const tran_low_t abs_dqc = abs(ctx->dqcoeff[ci]);
if (abs_qc == 1) {
- const int *base_cost = txb_costs->base_cost[coeff_ctx];
+ const int *const base_cost = txb_costs->base_cost[coeff_ctx];
const int rate = base_cost[1] + av1_cost_literal(1);
if (ctx->sharpness || abs_dqc < abs_tqc) {
return rate;
@@ -158,7 +158,7 @@ static int update_coeff_simple_nz(int ci, tran_low_t qc, int coeff_ctx,
const int64_t dist0 = get_coeff_dist_qm(abs_tqc, 0, shift, qm_val);
dist_diff_0 = get_coeff_dist_qm(abs_tqc, abs_dqc, shift, qm_val) - dist0;
}
- const int rate_low = rate - base_cost[5];
+ const int rate_low = base_cost[0];
const int64_t rdmult = ctx->rdmult;
const int64_t rd = RDCOST(rdmult, rate, dist_diff_0);
const int64_t rd_low = RDCOST(rdmult, rate_low, 0);
@@ -170,61 +170,58 @@ static int update_coeff_simple_nz(int ci, tran_low_t qc, int coeff_ctx,
}
return rate;
} else {
- if (abs_dqc < abs_tqc) {
- int rate = txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)] +
- av1_cost_literal(1);
- if (abs_qc > NUM_BASE_LEVELS) {
- const int br_ctx =
- get_br_ctx_simple(ctx->levels, ci, ctx->bhl, tx_class);
- rate += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]);
- }
- return rate;
- }
-
const int bhl = ctx->bhl;
uint8_t *levels = ctx->levels;
- int rate_low = 0;
- const int rate = get_two_coeff_cost_simple(
- ci, abs_qc, coeff_ctx, txb_costs, bhl, tx_class, levels, &rate_low);
-
- const int allow_lower_qc = ctx->sharpness ? (abs_qc > 1) : 1;
- if (allow_lower_qc) {
- // update_coeff_simple_nz is only invoked for AC coefficients (si > 0), so
- // dequant_ac (dequant[1]) can be used directly without a (ci > 0) branch.
- int dqv = ctx->dequant_ac;
- if (ctx->iqmatrix != NULL) {
- dqv = ((ctx->iqmatrix[ci] * dqv) + (1 << (AOM_QM_BITS - 1))) >>
- AOM_QM_BITS;
- }
- const int shift = ctx->shift;
- const tran_low_t abs_qc_low = abs_qc - 1;
- const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift;
- const qm_val_t *qmatrix = ctx->qmatrix;
- int64_t dist_diff_0, dist_diff_low_0;
- if (qmatrix == NULL) {
- const int64_t abs_tqc2 = (int64_t)abs_tqc << 1;
- dist_diff_0 =
- ((int64_t)abs_dqc * (abs_dqc - abs_tqc2)) * (1LL << (2 * shift));
- dist_diff_low_0 = ((int64_t)abs_dqc_low * (abs_dqc_low - abs_tqc2)) *
- (1LL << (2 * shift));
- } else {
- const int64_t qm_val = qmatrix[ci];
- const int64_t dist0 = get_coeff_dist_qm(abs_tqc, 0, shift, qm_val);
- dist_diff_0 =
- get_coeff_dist_qm(abs_tqc, abs_dqc, shift, qm_val) - dist0;
- dist_diff_low_0 =
- get_coeff_dist_qm(abs_tqc, abs_dqc_low, shift, qm_val) - dist0;
- }
- const int64_t rdmult = ctx->rdmult;
- const int64_t rd = RDCOST(rdmult, rate, dist_diff_0);
- const int64_t rd_low = RDCOST(rdmult, rate_low, dist_diff_low_0);
- if (rd_low < rd) {
- const int sign = (qc < 0) ? 1 : 0;
- ctx->qcoeff[ci] = (-sign ^ abs_qc_low) + sign;
- ctx->dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign;
- levels[get_padded_idx(ci, bhl)] = AOMMIN(abs_qc_low, INT8_MAX);
- return rate_low;
- }
+ int rate, rate_low;
+ if (abs_qc == 2) {
+ const int *const base_cost = txb_costs->base_cost[coeff_ctx];
+ rate = base_cost[2] + av1_cost_literal(1);
+ if (abs_dqc < abs_tqc) return rate;
+ rate_low = base_cost[1] + av1_cost_literal(1);
+ } else {
+ const int br_ctx = get_br_ctx_simple(levels, ci, bhl, tx_class);
+ const int *const lps_cost = txb_costs->lps_cost[br_ctx];
+ const int *const base_cost = txb_costs->base_cost[coeff_ctx];
+ const int base_rate = base_cost[3] + av1_cost_literal(1);
+ rate = base_rate + get_br_cost(abs_qc, lps_cost);
+ if (abs_dqc < abs_tqc) return rate;
+ rate_low = (abs_qc == 3)
+ ? (base_cost[2] + av1_cost_literal(1))
+ : (base_rate + get_br_cost(abs_qc - 1, lps_cost));
+ }
+
+ // update_coeff_simple_nz is only invoked for AC coefficients (si > 0), so
+ // dequant_ac (dequant[1]) can be used directly without a (ci > 0) branch.
+ int dqv = ctx->dequant_ac;
+ if (ctx->iqmatrix != NULL) {
+ dqv =
+ ((ctx->iqmatrix[ci] * dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
+ }
+ const int shift = ctx->shift;
+ const tran_low_t abs_qc_low = abs_qc - 1;
+ const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift;
+ const qm_val_t *qmatrix = ctx->qmatrix;
+ int64_t dist_diff_0, dist_diff_low_0;
+ if (qmatrix == NULL) {
+ const int64_t abs_tqc2 = (int64_t)abs_tqc << 1;
+ dist_diff_0 =
+ ((int64_t)abs_dqc * (abs_dqc - abs_tqc2)) * (1LL << (2 * shift));
+ dist_diff_low_0 = ((int64_t)abs_dqc_low * (abs_dqc_low - abs_tqc2)) *
+ (1LL << (2 * shift));
+ } else {
+ const int64_t qm_val = qmatrix[ci];
+ dist_diff_0 = get_coeff_dist_qm(abs_tqc, abs_dqc, shift, qm_val);
+ dist_diff_low_0 = get_coeff_dist_qm(abs_tqc, abs_dqc_low, shift, qm_val);
+ }
+ const int64_t rdmult = ctx->rdmult;
+ const int64_t rd = RDCOST(rdmult, rate, dist_diff_0);
+ const int64_t rd_low = RDCOST(rdmult, rate_low, dist_diff_low_0);
+ if (rd_low < rd) {
+ const int sign = (qc < 0) ? 1 : 0;
+ ctx->qcoeff[ci] = (-sign ^ abs_qc_low) + sign;
+ ctx->dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign;
+ levels[get_padded_idx(ci, bhl)] = AOMMIN(abs_qc_low, INT8_MAX);
+ return rate_low;
}
return rate;
}
@@ -253,8 +250,16 @@ static int update_coeff_eob_nz(int si, int ci, tran_low_t qc, int coeff_ctx,
const int allow_lower_qc = sharpness ? (abs_qc > qc_threshold) : 1;
const int allow_new_eob = (sharpness == 0 || new_eob >= min_eob_cutoff);
- int rate = get_coeff_cost_general(0, ci, abs_qc, sign, coeff_ctx, dc_sign_ctx,
- txb_costs, bhl, tx_class, levels);
+ const int sign_cost = (ci == 0) ? txb_costs->dc_sign_cost[dc_sign_ctx][sign]
+ : av1_cost_literal(1);
+ const int *const base_cost = txb_costs->base_cost[coeff_ctx];
+ const int *lps_cost = txb_costs->lps_cost[0];
+ int rate = base_cost[AOMMIN(abs_qc, 3)] + sign_cost;
+ if (abs_qc > NUM_BASE_LEVELS) {
+ const int br_ctx = get_br_ctx(levels, ci, bhl, tx_class);
+ lps_cost = txb_costs->lps_cost[br_ctx];
+ rate += get_br_cost(abs_qc, lps_cost);
+ }
int64_t dist;
int64_t dist0 = 0;
int64_t qm_val = 0;
@@ -297,11 +302,15 @@ static int update_coeff_eob_nz(int si, int ci, tran_low_t qc, int coeff_ctx,
int lower_level = 0;
if (allow_lower_qc) {
- const int rate_low =
- (abs_qc == 1) ? txb_costs->base_cost[coeff_ctx][0]
- : get_coeff_cost_general(
- 0, ci, abs_qc_low, sign, coeff_ctx, dc_sign_ctx,
- txb_costs, bhl, tx_class, levels);
+ int rate_low;
+ if (abs_qc == 1) {
+ rate_low = base_cost[0];
+ } else {
+ rate_low = base_cost[AOMMIN(abs_qc_low, 3)] + sign_cost;
+ if (abs_qc_low > NUM_BASE_LEVELS) {
+ rate_low += get_br_cost(abs_qc_low, lps_cost);
+ }
+ }
const int64_t rd_low =
RDCOST(rdmult, accu_rate + rate_low, accu_dist + dist_low);
if (rd_low < rd) {
@@ -317,18 +326,25 @@ static int update_coeff_eob_nz(int si, int ci, tran_low_t qc, int coeff_ctx,
const int coeff_ctx_new_eob = get_lower_levels_ctx_eob(bhl, ctx->width, si);
const int new_eob_cost =
get_eob_cost(new_eob, ctx->txb_eob_costs, txb_costs, tx_class);
+ const int *const base_eob_cost =
+ txb_costs->base_eob_cost[coeff_ctx_new_eob];
+ const int *lps_cost_eob = txb_costs->lps_cost[0];
int rate_coeff_eob =
- new_eob_cost + get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx_new_eob,
- dc_sign_ctx, txb_costs, bhl,
- tx_class);
+ new_eob_cost + base_eob_cost[AOMMIN(abs_qc, 3) - 1] + sign_cost;
+ if (abs_qc > NUM_BASE_LEVELS) {
+ const int br_ctx_eob = get_br_ctx_eob(ci, bhl, tx_class);
+ lps_cost_eob = txb_costs->lps_cost[br_ctx_eob];
+ rate_coeff_eob += get_br_cost(abs_qc, lps_cost_eob);
+ }
int64_t dist_new_eob = dist_orig;
int64_t rd_new_eob = RDCOST(rdmult, rate_coeff_eob, dist_new_eob);
if (abs_qc_low > 0) {
- const int rate_coeff_eob_low =
- new_eob_cost + get_coeff_cost_eob(ci, abs_qc_low, sign,
- coeff_ctx_new_eob, dc_sign_ctx,
- txb_costs, bhl, tx_class);
+ int rate_coeff_eob_low =
+ new_eob_cost + base_eob_cost[AOMMIN(abs_qc_low, 3) - 1] + sign_cost;
+ if (abs_qc_low > NUM_BASE_LEVELS) {
+ rate_coeff_eob_low += get_br_cost(abs_qc_low, lps_cost_eob);
+ }
const int64_t dist_new_eob_low = dist_low;
const int64_t rd_new_eob_low =
RDCOST(rdmult, rate_coeff_eob_low, dist_new_eob_low);
@@ -452,6 +468,9 @@ static AOM_FORCE_INLINE void update_coeff_eob_facade(
*eob, *nz_num, local_rate, bhl, width, dc_sign_ctx,
shift, sharpness, is_noise_pattern
};
+ const intptr_t stride = (1 << bhl) + TX_PAD_HOR;
+ const int8_t *const nz_map_ctx =
+ (tx_class == TX_CLASS_2D) ? av1_nz_map_ctx_offset[tx_size] : NULL;
const int (*base_cost)[8] = txb_costs->base_cost;
while (s >= 0 && ctx.nz_num <= max_nz_num) {
int ci;
@@ -459,8 +478,9 @@ static AOM_FORCE_INLINE void update_coeff_eob_facade(
int coeff_ctx;
do {
ci = scan[s];
+ coeff_ctx = get_lower_levels_ctx_fast(levels, ci, bhl, stride, nz_map_ctx,
+ tx_class);
qc = qcoeff[ci];
- coeff_ctx = get_lower_levels_ctx(levels, ci, bhl, tx_size, tx_class);
if (qc != 0) break;
local_rate += base_cost[coeff_ctx][0];
} while (--s >= 0);
@@ -491,6 +511,9 @@ static AOM_FORCE_INLINE void update_coeff_simple_facade(
const TxbSimpleCtx ctx = { rdmult, txb_costs, tcoeff, qcoeff,
dqcoeff, levels, iqmatrix, qmatrix,
bhl, shift, dequant[1], sharpness };
+ const intptr_t stride = (1 << bhl) + TX_PAD_HOR;
+ const int8_t *const nz_map_ctx =
+ (tx_class == TX_CLASS_2D) ? av1_nz_map_ctx_offset[tx_size] : NULL;
const int (*base_cost)[8] = txb_costs->base_cost;
while (s >= 1) {
int ci;
@@ -499,9 +522,9 @@ static AOM_FORCE_INLINE void update_coeff_simple_facade(
do {
ci = scan[s];
assert(ci > 0);
+ coeff_ctx = get_lower_levels_ctx_simple_fast(levels, ci, bhl, stride,
+ nz_map_ctx, tx_class);
qc = qcoeff[ci];
- coeff_ctx =
- get_lower_levels_ctx_simple(levels, ci, bhl, tx_size, tx_class);
if (qc != 0) break;
local_rate += base_cost[coeff_ctx][0];
} while (--s >= 1);
@@ -707,89 +730,16 @@ int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
return eob;
}
-static AOM_FORCE_INLINE int warehouse_efficients_txb(
- const MACROBLOCK *x, const int plane, const int block,
- const TX_SIZE tx_size, const TXB_CTX *const txb_ctx,
- const struct macroblock_plane *p, const int eob,
- const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs,
- const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class,
- int reduced_tx_set_used) {
- const tran_low_t *const __restrict qcoeff = p->qcoeff + BLOCK_OFFSET(block);
- const int txb_skip_ctx = txb_ctx->txb_skip_ctx;
- const int eob_multi_size = txsize_log2_minus4[tx_size];
- const LV_MAP_EOB_COST *const eob_costs =
- &x->coeff_costs.eob_costs[eob_multi_size][plane_type];
- int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0];
- cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used);
- cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class);
-
+static AOM_FORCE_INLINE int warehouse_efficients_txb_loop(
+ const tran_low_t *const __restrict qcoeff,
+ const int16_t *const __restrict scan, const int eob,
+ const uint8_t *const __restrict levels, const TX_SIZE tx_size,
+ const int bhl, const int width, const TXB_CTX *const txb_ctx,
+ const LV_MAP_COEFF_COST *const coeff_costs, const TX_CLASS tx_class,
+ int cost) {
const int (*lps_cost)[COEFF_BASE_RANGE + 1 + COEFF_BASE_RANGE + 1] =
coeff_costs->lps_cost;
-
- if (eob == 1) {
- const tran_low_t v = qcoeff[0];
- if (v) {
- const int sign = AOMSIGN(v);
- const int level = (v ^ sign) - sign;
- cost += coeff_costs->base_eob_cost[0][AOMMIN(level, 3) - 1];
- if (level > NUM_BASE_LEVELS) {
- cost += get_br_cost(level, lps_cost[0]);
- }
- const int sign01 = -sign;
- cost += coeff_costs->dc_sign_cost[txb_ctx->dc_sign_ctx][sign01];
- }
- return cost;
- }
-
- const int bhl = get_txb_bhl(tx_size);
- const int width = get_txb_wide(tx_size);
- const SCAN_ORDER *const scan_order = get_scan(tx_size, tx_type);
- const int16_t *const __restrict scan = scan_order->scan;
const int (*base_cost)[8] = coeff_costs->base_cost;
-
- if (eob == 2 && tx_class == TX_CLASS_2D) {
- const int pos1 = scan[1];
- const tran_low_t v1 = qcoeff[pos1];
- int level1 = 0;
- if (v1) {
- const int sign1 = AOMSIGN(v1);
- level1 = (v1 ^ sign1) - sign1;
- const int coeff_ctx1 = get_lower_levels_ctx_eob(bhl, width, 1);
- cost += coeff_costs->base_eob_cost[coeff_ctx1][AOMMIN(level1, 3) - 1];
- if (level1 > NUM_BASE_LEVELS) {
- const int ctx1 = get_br_ctx_eob(pos1, bhl, TX_CLASS_2D);
- cost += get_br_cost(level1, lps_cost[ctx1]);
- }
- cost += av1_cost_literal(1);
- }
- const tran_low_t v0 = qcoeff[0];
- if (!v0) {
- cost += base_cost[0][0];
- } else {
- const int sign0 = AOMSIGN(v0);
- const int level0 = (v0 ^ sign0) - sign0;
- cost += base_cost[0][AOMMIN(level0, 3)];
- const int sign01 = (sign0 ^ sign0) - sign0;
- const int dc_sign_ctx = txb_ctx->dc_sign_ctx;
- cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01];
- if (level0 > NUM_BASE_LEVELS) {
- const int mag =
- (pos1 == 1 || pos1 == (1 << bhl) || pos1 == (1 << bhl) + 1)
- ? AOMMIN(level1, INT8_MAX)
- : 0;
- const int ctx0 = AOMMIN((mag + 1) >> 1, 6);
- cost += get_br_cost(level0, lps_cost[ctx0]);
- }
- }
- return cost;
- }
-
- const int height = get_txb_high(tx_size);
- uint8_t levels_buf[TX_PAD_2D];
- uint8_t *const __restrict levels = set_levels(levels_buf, height);
-
- av1_txb_init_levels(qcoeff, width, height, levels);
-
int c = eob - 1;
{
const int pos = scan[c];
@@ -809,10 +759,13 @@ static AOM_FORCE_INLINE int warehouse_efficients_txb(
}
if (eob <= ((width << bhl) >> 4) + 3) {
+ const intptr_t stride = (1 << bhl) + TX_PAD_HOR;
+ const int8_t *const nz_map_ctx =
+ (tx_class == TX_CLASS_2D) ? av1_nz_map_ctx_offset[tx_size] : NULL;
for (c = eob - 2; c >= 1; --c) {
const int pos = scan[c];
- const int coeff_ctx =
- get_lower_levels_ctx_simple(levels, pos, bhl, tx_size, tx_class);
+ const int coeff_ctx = get_lower_levels_ctx_simple_fast(
+ levels, pos, bhl, stride, nz_map_ctx, tx_class);
const int level = abs(qcoeff[pos]);
if (level <= NUM_BASE_LEVELS) {
cost +=
@@ -834,9 +787,8 @@ static AOM_FORCE_INLINE int warehouse_efficients_txb(
const int sign = AOMSIGN(v);
const int level = (v ^ sign) - sign;
cost += base_cost[coeff_ctx][AOMMIN(level, 3)];
- const int sign01 = (sign ^ sign) - sign;
- const int dc_sign_ctx = txb_ctx->dc_sign_ctx;
- cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01];
+ const int sign01 = -sign;
+ cost += coeff_costs->dc_sign_cost[txb_ctx->dc_sign_ctx][sign01];
if (level > NUM_BASE_LEVELS) {
const int ctx = get_br_ctx(levels, 0, bhl, tx_class);
cost += get_br_cost(level, lps_cost[ctx]);
@@ -851,16 +803,16 @@ static AOM_FORCE_INLINE int warehouse_efficients_txb(
const int pos = scan[c];
const int coeff_ctx = coeff_contexts[pos];
const tran_low_t v = qcoeff[pos];
- if (!v) {
- cost += base_cost[coeff_ctx][0];
- continue;
- }
- const int level = abs(v);
- cost += base_cost[coeff_ctx][AOMMIN(level, 3)] + av1_cost_literal(1);
- if (level > NUM_BASE_LEVELS) {
- const int ctx = get_br_ctx_simple(levels, pos, bhl, tx_class);
- cost += get_br_cost(level, lps_cost[ctx]);
+ int c_cost = base_cost[coeff_ctx][0];
+ if (v) {
+ const int level = abs(v);
+ c_cost = base_cost[coeff_ctx][AOMMIN(level, 3)] + av1_cost_literal(1);
+ if (level > NUM_BASE_LEVELS) {
+ const int ctx = get_br_ctx_simple(levels, pos, bhl, tx_class);
+ c_cost += get_br_cost(level, lps_cost[ctx]);
+ }
}
+ cost += c_cost;
}
// c == 0 after previous loop
{
@@ -874,9 +826,8 @@ static AOM_FORCE_INLINE int warehouse_efficients_txb(
const int level = (v ^ sign) - sign;
cost += base_cost[coeff_ctx][AOMMIN(level, 3)];
// sign bit cost
- const int sign01 = (sign ^ sign) - sign;
- const int dc_sign_ctx = txb_ctx->dc_sign_ctx;
- cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01];
+ const int sign01 = -sign;
+ cost += coeff_costs->dc_sign_cost[txb_ctx->dc_sign_ctx][sign01];
if (level > NUM_BASE_LEVELS) {
const int ctx = get_br_ctx(levels, pos, bhl, tx_class);
cost += get_br_cost(level, lps_cost[ctx]);
@@ -886,6 +837,41 @@ static AOM_FORCE_INLINE int warehouse_efficients_txb(
return cost;
}
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
+static int warehouse_efficients_txb_multi(
+ const tran_low_t *const qcoeff, const TX_SIZE tx_size,
+ const TXB_CTX *const txb_ctx, const int eob,
+ const LV_MAP_COEFF_COST *const coeff_costs, const TX_TYPE tx_type,
+ const TX_CLASS tx_class, int cost) {
+ const int bhl = get_txb_bhl(tx_size);
+ const int width = get_txb_wide(tx_size);
+ const int height = get_txb_high(tx_size);
+ const SCAN_ORDER *const scan_order = get_scan(tx_size, tx_type);
+ const int16_t *const scan = scan_order->scan;
+ uint8_t levels_buf[TX_PAD_2D];
+ uint8_t *const levels = set_levels(levels_buf, height);
+
+ av1_txb_init_levels(qcoeff, width, height, levels);
+
+ switch (tx_class) {
+ case TX_CLASS_2D:
+ return warehouse_efficients_txb_loop(qcoeff, scan, eob, levels, tx_size,
+ bhl, width, txb_ctx, coeff_costs,
+ TX_CLASS_2D, cost);
+ case TX_CLASS_VERT:
+ return warehouse_efficients_txb_loop(qcoeff, scan, eob, levels, tx_size,
+ bhl, width, txb_ctx, coeff_costs,
+ TX_CLASS_VERT, cost);
+ case TX_CLASS_HORIZ:
+ default:
+ return warehouse_efficients_txb_loop(qcoeff, scan, eob, levels, tx_size,
+ bhl, width, txb_ctx, coeff_costs,
+ TX_CLASS_HORIZ, cost);
+ }
+}
+
/*!\brief Estimate the entropy cost of transform coefficients using Laplacian
* distribution.
*
@@ -988,25 +974,74 @@ int av1_cost_coeffs_txb(const MACROBLOCK *x, const int plane, const int block,
const MACROBLOCKD *const xd = &x->e_mbd;
const TX_CLASS tx_class = tx_type_to_class[tx_type];
+ const tran_low_t *const __restrict qcoeff = p->qcoeff + BLOCK_OFFSET(block);
+ const int txb_skip_ctx = txb_ctx->txb_skip_ctx;
+ const int eob_multi_size = txsize_log2_minus4[tx_size];
+ const LV_MAP_EOB_COST *const eob_costs =
+ &x->coeff_costs.eob_costs[eob_multi_size][plane_type];
+ int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0];
+ cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used);
+ cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class);
- switch (tx_class) {
- case TX_CLASS_2D:
- return warehouse_efficients_txb(x, plane, block, tx_size, txb_ctx, p, eob,
- plane_type, coeff_costs, xd, tx_type,
- TX_CLASS_2D, reduced_tx_set_used);
-
- case TX_CLASS_VERT:
- return warehouse_efficients_txb(x, plane, block, tx_size, txb_ctx, p, eob,
- plane_type, coeff_costs, xd, tx_type,
- TX_CLASS_VERT, reduced_tx_set_used);
-
- case TX_CLASS_HORIZ:
- return warehouse_efficients_txb(x, plane, block, tx_size, txb_ctx, p, eob,
- plane_type, coeff_costs, xd, tx_type,
- TX_CLASS_HORIZ, reduced_tx_set_used);
+ if (eob == 1) {
+ const tran_low_t v = qcoeff[0];
+ if (v) {
+ const int sign = AOMSIGN(v);
+ const int level = (v ^ sign) - sign;
+ cost += coeff_costs->base_eob_cost[0][AOMMIN(level, 3) - 1];
+ if (level > NUM_BASE_LEVELS) {
+ cost += get_br_cost(level, coeff_costs->lps_cost[0]);
+ }
+ const int sign01 = -sign;
+ cost += coeff_costs->dc_sign_cost[txb_ctx->dc_sign_ctx][sign01];
+ }
+ return cost;
+ }
- default: assert(0 && "Invalid TX_CLASS"); return 0;
+ if (eob == 2 && tx_class == TX_CLASS_2D) {
+ const int bhl = get_txb_bhl(tx_size);
+ const int width = get_txb_wide(tx_size);
+ const SCAN_ORDER *const scan_order = get_scan(tx_size, tx_type);
+ const int (*lps_cost)[COEFF_BASE_RANGE + 1 + COEFF_BASE_RANGE + 1] =
+ coeff_costs->lps_cost;
+ const int (*base_cost)[8] = coeff_costs->base_cost;
+ const int pos1 = scan_order->scan[1];
+ const tran_low_t v1 = qcoeff[pos1];
+ int level1 = 0;
+ if (v1) {
+ const int sign1 = AOMSIGN(v1);
+ level1 = (v1 ^ sign1) - sign1;
+ const int coeff_ctx1 = get_lower_levels_ctx_eob(bhl, width, 1);
+ cost += coeff_costs->base_eob_cost[coeff_ctx1][AOMMIN(level1, 3) - 1];
+ if (level1 > NUM_BASE_LEVELS) {
+ const int ctx1 = get_br_ctx_eob(pos1, bhl, TX_CLASS_2D);
+ cost += get_br_cost(level1, lps_cost[ctx1]);
+ }
+ cost += av1_cost_literal(1);
+ }
+ const tran_low_t v0 = qcoeff[0];
+ if (!v0) {
+ cost += base_cost[0][0];
+ } else {
+ const int sign0 = AOMSIGN(v0);
+ const int level0 = (v0 ^ sign0) - sign0;
+ cost += base_cost[0][AOMMIN(level0, 3)];
+ const int sign01 = -sign0;
+ cost += coeff_costs->dc_sign_cost[txb_ctx->dc_sign_ctx][sign01];
+ if (level0 > NUM_BASE_LEVELS) {
+ const int mag =
+ (pos1 == 1 || pos1 == (1 << bhl) || pos1 == (1 << bhl) + 1)
+ ? AOMMIN(level1, INT8_MAX)
+ : 0;
+ const int ctx0 = AOMMIN((mag + 1) >> 1, 6);
+ cost += get_br_cost(level0, lps_cost[ctx0]);
+ }
+ }
+ return cost;
}
+
+ return warehouse_efficients_txb_multi(qcoeff, tx_size, txb_ctx, eob,
+ coeff_costs, tx_type, tx_class, cost);
}
int av1_cost_coeffs_txb_laplacian(const MACROBLOCK *x, const int plane,
diff --git a/test/subtract_test.cc b/test/subtract_test.cc
index 9d3e059682..2522db063a 100644
--- a/test/subtract_test.cc
+++ b/test/subtract_test.cc
@@ -60,25 +60,23 @@ class AV1SubtractBlockTestBase : public ::testing::Test {
void SetUp() override {
rnd_.Reset(ACMRandom::DeterministicSeed());
- const size_t max_width = 128;
- const size_t max_block_size = max_width * max_width;
if (hbd_) {
src_ = CONVERT_TO_BYTEPTR(reinterpret_cast<uint16_t *>(
- aom_memalign(16, max_block_size * sizeof(uint16_t))));
+ aom_memalign(32, kMaxBufElems * sizeof(uint16_t))));
ASSERT_NE(src_, nullptr);
pred_ = CONVERT_TO_BYTEPTR(reinterpret_cast<uint16_t *>(
- aom_memalign(16, max_block_size * sizeof(uint16_t))));
+ aom_memalign(32, kMaxBufElems * sizeof(uint16_t))));
ASSERT_NE(pred_, nullptr);
} else {
src_ = reinterpret_cast<uint8_t *>(
- aom_memalign(16, max_block_size * sizeof(uint8_t)));
+ aom_memalign(32, kMaxBufElems * sizeof(uint8_t)));
ASSERT_NE(src_, nullptr);
pred_ = reinterpret_cast<uint8_t *>(
- aom_memalign(16, max_block_size * sizeof(uint8_t)));
+ aom_memalign(32, kMaxBufElems * sizeof(uint8_t)));
ASSERT_NE(pred_, nullptr);
}
diff_ = reinterpret_cast<int16_t *>(
- aom_memalign(32, max_block_size * sizeof(int16_t)));
+ aom_memalign(32, kMaxBufElems * sizeof(int16_t)));
ASSERT_NE(diff_, nullptr);
}
@@ -94,11 +92,17 @@ class AV1SubtractBlockTestBase : public ::testing::Test {
}
protected:
+ static constexpr int kMaxHeight = 128;
+ static constexpr int kMaxStride = 256;
+ static constexpr int kMaxOffset = 64;
+ static constexpr size_t kMaxBufElems =
+ static_cast<size_t>(kMaxHeight * kMaxStride + kMaxOffset);
+
void CheckResult();
void RunForSpeed();
private:
- void FillInputs();
+ void FillInputs(size_t count);
ACMRandom rnd_;
int block_height_;
@@ -112,20 +116,17 @@ class AV1SubtractBlockTestBase : public ::testing::Test {
int16_t *diff_;
};
-void AV1SubtractBlockTestBase::FillInputs() {
- const size_t max_width = 128;
- const int max_block_size = max_width * max_width;
+void AV1SubtractBlockTestBase::FillInputs(size_t count) {
if (hbd_) {
const int mask = (1 << bit_depth_) - 1;
- for (int i = 0; i < max_block_size; ++i) {
- CONVERT_TO_SHORTPTR(src_)[i] = rnd_.Rand16() & mask;
- CONVERT_TO_SHORTPTR(pred_)[i] = rnd_.Rand16() & mask;
+ uint16_t *const src16 = CONVERT_TO_SHORTPTR(src_);
+ uint16_t *const pred16 = CONVERT_TO_SHORTPTR(pred_);
+ for (size_t i = 0; i < count; ++i) {
+ src16[i] = rnd_.Rand16() & mask;
+ pred16[i] = rnd_.Rand16() & mask;
}
} else {
- if (src_ == nullptr) {
- std::cerr << "gadfg" << std::endl;
- }
- for (int i = 0; i < max_block_size; ++i) {
+ for (size_t i = 0; i < count; ++i) {
src_[i] = rnd_.Rand8();
pred_[i] = rnd_.Rand8();
}
@@ -134,30 +135,97 @@ void AV1SubtractBlockTestBase::FillInputs() {
void AV1SubtractBlockTestBase::CheckResult() {
const int test_num = 100;
- int i;
-
- for (i = 0; i < test_num; ++i) {
- FillInputs();
+ // Encoder residual sub-blocks keep diff_ptr and diff_stride aligned to
+ // min(cols, 16) int16_t elements (8B for cols==4, 16B for cols==8, 32B for
+ // cols>=16).
+ const int diff_align = (block_width_ < 16) ? block_width_ : 16;
+ bool sse2_aligned_src = false;
+#if HAVE_SSE2 && HAVE_X86_ASM
+ if (!hbd_ && func_ == &aom_subtract_block_sse2 && block_width_ >= 16) {
+ sse2_aligned_src = true;
+ }
+#endif
- func_(block_height_, block_width_, diff_, block_width_, src_, block_width_,
- pred_, block_width_);
+ for (int i = 0; i < test_num; ++i) {
+ ptrdiff_t src_stride = block_width_;
+ ptrdiff_t pred_stride = block_width_;
+ ptrdiff_t diff_stride = block_width_;
+ int src_off = 0;
+ int pred_off = 0;
+ int diff_off = 0;
+ if (i > 0) {
+ if (sse2_aligned_src) {
+ src_stride = block_width_ + (rnd_(5) * 16);
+ src_off = rnd_(4) * 16;
+ } else {
+ src_stride = block_width_ + rnd_(65);
+ src_off = rnd_(33);
+ }
+ pred_stride = block_width_ + rnd_(65);
+ pred_off = rnd_(33);
+ // Ensure diff_stride >= diff_align so cols<16 sub-block strides can also
+ // be wider (e.g. diff_stride == 16..192 for cols == 4 or 8).
+ diff_stride = ((block_width_ + diff_align - 1) & ~(diff_align - 1)) +
+ rnd_(5) * diff_align;
+ diff_off = rnd_(4) * diff_align;
+ }
- if (hbd_)
+ const size_t src_elems =
+ static_cast<size_t>(src_off + block_height_ * src_stride);
+ const size_t pred_elems =
+ static_cast<size_t>(pred_off + block_height_ * pred_stride);
+ const size_t diff_elems =
+ static_cast<size_t>(diff_off + block_height_ * diff_stride);
+ ASSERT_LE(src_elems, kMaxBufElems);
+ ASSERT_LE(pred_elems, kMaxBufElems);
+ ASSERT_LE(diff_elems, kMaxBufElems);
+
+ FillInputs(src_elems > pred_elems ? src_elems : pred_elems);
+ static constexpr int16_t kDiffCanary = 0x7abc;
+ for (size_t k = 0; k < diff_elems; ++k) diff_[k] = kDiffCanary;
+
+ const uint8_t *const src_ptr =
+ hbd_ ? CONVERT_TO_BYTEPTR(CONVERT_TO_SHORTPTR(src_) + src_off)
+ : (src_ + src_off);
+ const uint8_t *const pred_ptr =
+ hbd_ ? CONVERT_TO_BYTEPTR(CONVERT_TO_SHORTPTR(pred_) + pred_off)
+ : (pred_ + pred_off);
+ int16_t *const diff_ptr = diff_ + diff_off;
+
+ API_REGISTER_STATE_CHECK(func_(block_height_, block_width_, diff_ptr,
+ diff_stride, src_ptr, src_stride, pred_ptr,
+ pred_stride));
+
+ for (int k = 0; k < diff_off; ++k) {
+ ASSERT_EQ(diff_[k], kDiffCanary)
+ << "clobbered before diff_ptr, test " << i;
+ }
+ if (hbd_) {
+ const uint16_t *const s16 = CONVERT_TO_SHORTPTR(src_ptr);
+ const uint16_t *const p16 = CONVERT_TO_SHORTPTR(pred_ptr);
for (int r = 0; r < block_height_; ++r) {
for (int c = 0; c < block_width_; ++c) {
- EXPECT_EQ(diff_[r * block_width_ + c],
- (CONVERT_TO_SHORTPTR(src_)[r * block_width_ + c] -
- CONVERT_TO_SHORTPTR(pred_)[r * block_width_ + c]))
+ ASSERT_EQ(diff_ptr[r * diff_stride + c],
+ static_cast<int16_t>(s16[r * src_stride + c] -
+ p16[r * pred_stride + c]))
<< "r = " << r << ", c = " << c << ", test: " << i;
}
+ for (ptrdiff_t c = block_width_; c < diff_stride; ++c) {
+ ASSERT_EQ(diff_ptr[r * diff_stride + c], kDiffCanary)
+ << "padding clobbered r = " << r << ", c = " << c;
+ }
}
- else {
+ } else {
for (int r = 0; r < block_height_; ++r) {
for (int c = 0; c < block_width_; ++c) {
- EXPECT_EQ(diff_[r * block_width_ + c],
- src_[r * block_width_ + c] - pred_[r * block_width_ + c])
+ ASSERT_EQ(diff_ptr[r * diff_stride + c],
+ src_ptr[r * src_stride + c] - pred_ptr[r * pred_stride + c])
<< "r = " << r << ", c = " << c << ", test: " << i;
}
+ for (ptrdiff_t c = block_width_; c < diff_stride; ++c) {
+ ASSERT_EQ(diff_ptr[r * diff_stride + c], kDiffCanary)
+ << "padding clobbered r = " << r << ", c = " << c;
+ }
}
}
}
@@ -169,7 +237,8 @@ void AV1SubtractBlockTestBase::RunForSpeed() {
if (ref_func_ == func_) GTEST_SKIP();
- FillInputs();
+ const size_t block_elems = static_cast<size_t>(block_width_ * block_height_);
+ FillInputs(block_elems);
aom_usec_timer ref_timer;
aom_usec_timer_start(&ref_timer);
@@ -180,7 +249,7 @@ void AV1SubtractBlockTestBase::RunForSpeed() {
aom_usec_timer_mark(&ref_timer);
const int64_t ref_elapsed_time = aom_usec_timer_elapsed(&ref_timer);
- FillInputs();
+ FillInputs(block_elems);
aom_usec_timer timer;
aom_usec_timer_start(&timer);
@@ -213,12 +282,13 @@ GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(AV1SubtractBlockTest);
TEST_P(AV1SubtractBlockTest, CheckResult) { CheckResult(); }
TEST_P(AV1SubtractBlockTest, DISABLED_Speed) { RunForSpeed(); }
-const BLOCK_SIZE kValidBlockSize[] = { BLOCK_4X4, BLOCK_4X8, BLOCK_8X4,
- BLOCK_8X8, BLOCK_8X16, BLOCK_16X8,
- BLOCK_16X16, BLOCK_16X32, BLOCK_32X16,
- BLOCK_32X32, BLOCK_32X64, BLOCK_64X32,
- BLOCK_64X64, BLOCK_64X128, BLOCK_128X64,
- BLOCK_128X128 };
+const BLOCK_SIZE kValidBlockSize[] = {
+ BLOCK_4X4, BLOCK_4X8, BLOCK_8X4, BLOCK_8X8, BLOCK_8X16,
+ BLOCK_16X8, BLOCK_16X16, BLOCK_16X32, BLOCK_32X16, BLOCK_32X32,
+ BLOCK_32X64, BLOCK_64X32, BLOCK_64X64, BLOCK_64X128, BLOCK_128X64,
+ BLOCK_128X128, BLOCK_4X16, BLOCK_16X4, BLOCK_8X32, BLOCK_32X8,
+ BLOCK_16X64, BLOCK_64X16
+};
INSTANTIATE_TEST_SUITE_P(
C, AV1SubtractBlockTest,