Commit c19d2f07f2 for aom
commit c19d2f07f2101173bffe84f6447fc953518f739f
Author: Lin Zheng <linzhen@google.com>
Date: Sun Nov 8 23:00:39 2026 +0000
Highway: Add AVX512 obmc_var & comp_avg/mask_pred
Function / Block Size │ AVX2 (ns) │ AVX512 (ns) │ Speedup (x)
────────────────────────┼───────────┼─────────────┼────────────
obmc_variance 16x8 │ 19.87 │ 11.37 │ 1.75x
obmc_variance 16x16 │ 37.92 │ 19.57 │ 1.94x
obmc_variance 16x32 │ 71.62 │ 45.00 │ 1.59x
obmc_variance 32x16 │ 69.29 │ 41.55 │ 1.67x
obmc_variance 32x32 │ 133.90 │ 76.72 │ 1.75x
obmc_variance 32x64 │ 264.55 │ 145.39 │ 1.82x
obmc_variance 64x32 │ 257.34 │ 140.64 │ 1.83x
obmc_variance 64x64 │ 520.31 │ 283.27 │ 1.84x
obmc_variance 64x128 │ 1028.12 │ 622.77 │ 1.65x
obmc_variance 128x64 │ 1024.66 │ 565.44 │ 1.81x
obmc_variance 128x128 │ 2015.57 │ 1072.11 │ 1.88x
comp_avg_pred 32x32 │ 12.57 │ 8.16 │ 1.54x
comp_avg_pred 64x64 │ 58.28 │ 31.46 │ 1.85x
comp_avg_pred 128x128 │ 204.89 │ 96.21 │ 2.13x
comp_mask_pred 32x32 │ 40.82 │ 23.45 │ 1.74x
comp_mask_pred 64x64 │ 111.31 │ 67.04 │ 1.66x
comp_mask_pred 128x128 │ 457.80 │ 379.88 │ 1.21x
Change-Id: Ib00bfd0c986bf1cc92fa90a0021b55c5ec775873
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index 7358e4d21f..3c38010ef4 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -1837,6 +1837,9 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/unsigned int/, "aom_obmc_variance${w}x${h}", "const uint8_t *pre, int pre_stride, const int32_t *wsrc, const int32_t *mask, unsigned int *sse";
add_proto qw/unsigned int/, "aom_obmc_sub_pixel_variance${w}x${h}", "const uint8_t *pre, int pre_stride, int xoffset, int yoffset, const int32_t *wsrc, const int32_t *mask, unsigned int *sse";
specialize "aom_obmc_variance${w}x${h}", qw/sse4_1 avx2 neon/;
+ if (aom_config("CONFIG_HIGHWAY") eq "yes" && $w >= 16) {
+ specialize "aom_obmc_variance${w}x${h}", qw/avx512/;
+ }
specialize "aom_obmc_sub_pixel_variance${w}x${h}", qw/sse4_1 neon/;
}
@@ -1858,6 +1861,9 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
#
add_proto qw/void aom_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride";
specialize qw/aom_comp_avg_pred avx2 neon/;
+ if (aom_config("CONFIG_HIGHWAY") eq "yes") {
+ specialize qw/aom_comp_avg_pred avx512/;
+ }
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/void aom_highbd_comp_avg_pred/, "uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride";
@@ -1869,10 +1875,16 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/void aom_comp_mask_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask";
specialize qw/aom_comp_mask_pred ssse3 avx2 neon/;
+ if (aom_config("CONFIG_HIGHWAY") eq "yes") {
+ specialize qw/aom_comp_mask_pred avx512/;
+ }
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/void aom_highbd_comp_mask_pred/, "uint8_t *comp_pred, const uint8_t *pred8, int width, int height, const uint8_t *ref8, int ref_stride, const uint8_t *mask, int mask_stride, int invert_mask";
specialize qw/aom_highbd_comp_mask_pred sse2 avx2 neon/;
+ if (aom_config("CONFIG_HIGHWAY") eq "yes") {
+ specialize qw/aom_highbd_comp_mask_pred avx512/;
+ }
}
# Flow estimation library
diff --git a/aom_dsp/variance_hwy.h b/aom_dsp/variance_hwy.h
index 4d5331529e..6ce320ee28 100644
--- a/aom_dsp/variance_hwy.h
+++ b/aom_dsp/variance_hwy.h
@@ -19,6 +19,8 @@
#include "config/aom_config.h"
#include "config/aom_dsp_rtcd.h"
+#include "aom_dsp/aom_dsp_common.h"
+#include "aom_dsp/blend.h"
#include "aom_dsp/reduce_sum_hwy.h"
#include "third_party/highway/hwy/highway.h"
@@ -704,6 +706,281 @@ HWY_ATTR HWY_INLINE unsigned int SubPixelVarianceWxH(
(WLog2 + HLog2));
}
+// -----------------------------------------------------------------------------
+// OBMC Variance & CompoundAvg / CompoundMask Pred
+// -----------------------------------------------------------------------------
+
+#if !CONFIG_REALTIME_ONLY
+HWY_ATTR HWY_INLINE void ObmcVarianceW16N(
+ const uint8_t *pre, const int pre_stride, const int32_t *wsrc,
+ const int32_t *mask, unsigned int *const sse, int *const sum, const int w,
+ const int h) {
+ constexpr hn::ScalableTag<int32_t> di32;
+ constexpr hn::Repartition<int16_t, decltype(di32)> di16;
+ constexpr hn::Rebind<uint8_t, decltype(di32)> du8_quarter;
+ constexpr int kStep = hn::MaxLanes(di32);
+
+ assert(w >= kStep);
+ assert(IS_POWER_OF_TWO(w));
+ assert(IS_POWER_OF_TWO(h));
+
+ int n = 0;
+ const auto v_bias_d = hn::Set(di32, (1 << 12) >> 1);
+ auto v_sum_d = hn::Zero(di32);
+ auto v_sse_d = hn::Zero(di32);
+
+ for (int row = 0; row < h; ++row) {
+ for (int col = 0; col < w; col += kStep) {
+ const auto v_p_b = hn::LoadU(du8_quarter, pre + col);
+ const auto v_m_d = hn::LoadU(di32, mask + n);
+ const auto v_w_d = hn::LoadU(di32, wsrc + n);
+ const auto v_p_d = hn::PromoteTo(di32, v_p_b);
+ // Both pre ([0, 255]) and mask ([0, 64 * 64]) fit in signed 15-bit
+ // integers, so their upper int16_t halves in int32_t lanes are 0 and
+ // WidenMulPairwiseAdd computes exact 32-bit products pre * mask.
+ const auto v_pm_d = hn::WidenMulPairwiseAdd(
+ di32, hn::BitCast(di16, v_p_d), hn::BitCast(di16, v_m_d));
+ const auto v_diff_d = hn::Sub(v_w_d, v_pm_d);
+ const auto v_sign_d = hn::ShiftRight<31>(v_diff_d);
+ const auto v_tmp_d = hn::Add(hn::Add(v_diff_d, v_bias_d), v_sign_d);
+ const auto v_rdiff_d = hn::ShiftRight<12>(v_tmp_d);
+
+ v_sum_d = hn::Add(v_sum_d, v_rdiff_d);
+ v_sse_d = hn::Add(v_sse_d, hn::Mul(v_rdiff_d, v_rdiff_d));
+ n += kStep;
+ }
+ pre += pre_stride;
+ }
+ *sum = VarianceFinalFrom32BitSum(di32, v_sse_d, v_sum_d, sse);
+}
+#endif // !CONFIG_REALTIME_ONLY
+
+HWY_ATTR HWY_INLINE void CompAvgPred(uint8_t *comp_pred, const uint8_t *pred,
+ int width, int height, const uint8_t *ref,
+ int ref_stride) {
+ constexpr hn::ScalableTag<uint8_t> du8;
+ constexpr int vw = hn::MaxLanes(du8);
+ int row = 0;
+ if (width == 8) {
+ assert(height % 4 == 0);
+ constexpr hn::FixedTag<uint8_t, 32> du8_32;
+ do {
+ const auto pred_0123 = hn::LoadU(du8_32, pred);
+ const auto ref_0123 = LoadRowsU<4>(du8_32, ref, ref_stride);
+ hn::StoreU(hn::AverageRound(pred_0123, ref_0123), du8_32, comp_pred);
+ row += 4;
+ pred += 32;
+ comp_pred += 32;
+ ref += 4 * ref_stride;
+ } while (row < height);
+ } else if (width == 16) {
+ constexpr int kRows = (16 < vw) ? (vw / 16) : 1;
+ do {
+ const auto pred_vec = hn::LoadU(du8, pred);
+ const auto ref_vec = LoadRowsU<kRows>(du8, ref, ref_stride);
+ hn::StoreU(hn::AverageRound(pred_vec, ref_vec), du8, comp_pred);
+ row += kRows;
+ pred += 16 * kRows;
+ comp_pred += 16 * kRows;
+ ref += kRows * ref_stride;
+ } while (row < height);
+ } else if (width == 32) {
+ constexpr int kRows = (32 < vw) ? (vw / 32) : 1;
+ do {
+ const auto pred_vec = hn::LoadU(du8, pred);
+ const auto ref_vec = LoadRowsU<kRows>(du8, ref, ref_stride);
+ hn::StoreU(hn::AverageRound(pred_vec, ref_vec), du8, comp_pred);
+ row += kRows;
+ pred += 32 * kRows;
+ comp_pred += 32 * kRows;
+ ref += kRows * ref_stride;
+ } while (row < height);
+ } else if (width % vw == 0) {
+ do {
+ for (int x = 0; x < width; x += vw) {
+ const auto p0 = hn::LoadU(du8, pred + x);
+ const auto r0 = hn::LoadU(du8, ref + x);
+ const auto p1 = hn::LoadU(du8, pred + width + x);
+ const auto r1 = hn::LoadU(du8, ref + ref_stride + x);
+ hn::StoreU(hn::AverageRound(p0, r0), du8, comp_pred + x);
+ hn::StoreU(hn::AverageRound(p1, r1), du8, comp_pred + width + x);
+ }
+ row += 2;
+ pred += 2 * width;
+ comp_pred += 2 * width;
+ ref += 2 * ref_stride;
+ } while (row < height);
+ } else {
+ aom_comp_avg_pred_c(comp_pred, pred, width, height, ref, ref_stride);
+ }
+}
+
+template <class DU8>
+HWY_ATTR HWY_INLINE void CompMaskPredLine(DU8 du8, hn::VFromD<DU8> s0,
+ hn::VFromD<DU8> s1, hn::VFromD<DU8> a,
+ uint8_t *comp_pred) {
+ constexpr hn::Repartition<int8_t, DU8> di8;
+ constexpr hn::Repartition<int16_t, DU8> di16;
+ const auto alpha_max =
+ hn::Set(du8, static_cast<uint8_t>(AOM_BLEND_A64_MAX_ALPHA));
+ constexpr int16_t round_bits = 15 - AOM_BLEND_A64_ROUND_BITS;
+ const auto round_offset =
+ hn::Set(di16, static_cast<int16_t>(1 << round_bits));
+ const auto ma = hn::Sub(alpha_max, a);
+
+ const auto ssAL = hn::InterleaveLower(du8, s0, s1);
+ const auto aaAL = hn::BitCast(di8, hn::InterleaveLower(du8, a, ma));
+ const auto ssAH = hn::InterleaveUpper(du8, s0, s1);
+ const auto aaAH = hn::BitCast(di8, hn::InterleaveUpper(du8, a, ma));
+
+ const auto blendAL = hn::SatWidenMulPairwiseAdd(di16, ssAL, aaAL);
+ const auto blendAH = hn::SatWidenMulPairwiseAdd(di16, ssAH, aaAH);
+ const auto roundAL = hn::MulFixedPoint15(blendAL, round_offset);
+ const auto roundAH = hn::MulFixedPoint15(blendAH, round_offset);
+
+ const auto roundA = hn::ReorderDemote2To(du8, roundAL, roundAH);
+ hn::StoreU(roundA, du8, comp_pred);
+}
+
+HWY_ATTR HWY_INLINE void CompMaskPred(uint8_t *comp_pred, const uint8_t *pred,
+ int width, int height, const uint8_t *ref,
+ int ref_stride, const uint8_t *mask,
+ int mask_stride, int invert_mask) {
+ constexpr hn::ScalableTag<uint8_t> du8;
+ constexpr int vw = hn::MaxLanes(du8);
+ int i = 0;
+ const uint8_t *src0 = invert_mask ? pred : ref;
+ const uint8_t *src1 = invert_mask ? ref : pred;
+ const int stride0 = invert_mask ? width : ref_stride;
+ const int stride1 = invert_mask ? ref_stride : width;
+
+ if (width == 8) {
+ aom_comp_mask_pred_avx2(comp_pred, pred, width, height, ref, ref_stride,
+ mask, mask_stride, invert_mask);
+ } else if (width == 16) {
+ constexpr int kRows = (16 < vw) ? (vw / 16) : 1;
+ do {
+ const auto s0 = LoadRowsU<kRows>(du8, src0, stride0);
+ const auto s1 = LoadRowsU<kRows>(du8, src1, stride1);
+ const auto a = LoadRowsU<kRows>(du8, mask, mask_stride);
+ CompMaskPredLine(du8, s0, s1, a, comp_pred);
+ src0 += stride0 * kRows;
+ src1 += stride1 * kRows;
+ mask += mask_stride * kRows;
+ comp_pred += 16 * kRows;
+ i += kRows;
+ } while (i < height);
+ } else if (width == 32) {
+ constexpr int kRows = (32 < vw) ? (vw / 32) : 1;
+ do {
+ const auto s0 = LoadRowsU<kRows>(du8, src0, stride0);
+ const auto s1 = LoadRowsU<kRows>(du8, src1, stride1);
+ const auto a = LoadRowsU<kRows>(du8, mask, mask_stride);
+ CompMaskPredLine(du8, s0, s1, a, comp_pred);
+ src0 += stride0 * kRows;
+ src1 += stride1 * kRows;
+ mask += mask_stride * kRows;
+ comp_pred += 32 * kRows;
+ i += kRows;
+ } while (i < height);
+ } else {
+ do {
+ for (int x = 0; x < width; x += vw) {
+ const auto s0 = hn::LoadU(du8, src0 + x);
+ const auto s1 = hn::LoadU(du8, src1 + x);
+ const auto a = hn::LoadU(du8, mask + x);
+ CompMaskPredLine(du8, s0, s1, a, comp_pred);
+ comp_pred += vw;
+ }
+ src0 += stride0;
+ src1 += stride1;
+ mask += mask_stride;
+ i++;
+ } while (i < height);
+ }
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+template <class DI16>
+HWY_ATTR HWY_INLINE hn::VFromD<DI16> HighbdCompMaskPredLine(
+ DI16 di16, hn::VFromD<DI16> s0, hn::VFromD<DI16> s1, hn::VFromD<DI16> a) {
+ constexpr hn::Repartition<int32_t, DI16> di32;
+ const auto alpha_max =
+ hn::Set(di16, static_cast<int16_t>(1 << AOM_BLEND_A64_ROUND_BITS));
+ const auto round_const = hn::Set(di32, (1 << AOM_BLEND_A64_ROUND_BITS) >> 1);
+ const auto a_inv = hn::Sub(alpha_max, a);
+
+ const auto s_lo = hn::InterleaveLower(di16, s0, s1);
+ const auto a_lo = hn::InterleaveLower(di16, a, a_inv);
+ const auto pred_l = hn::ShiftRight<AOM_BLEND_A64_ROUND_BITS>(
+ hn::Add(hn::WidenMulPairwiseAdd(di32, s_lo, a_lo), round_const));
+
+ const auto s_hi = hn::InterleaveUpper(di16, s0, s1);
+ const auto a_hi = hn::InterleaveUpper(di16, a, a_inv);
+ const auto pred_h = hn::ShiftRight<AOM_BLEND_A64_ROUND_BITS>(
+ hn::Add(hn::WidenMulPairwiseAdd(di32, s_hi, a_hi), round_const));
+
+ return hn::ReorderDemote2To(di16, pred_l, pred_h);
+}
+
+HWY_ATTR HWY_INLINE void HighbdCompMaskPred(uint8_t *comp_pred8,
+ const uint8_t *pred8, int width,
+ int height, const uint8_t *ref8,
+ int ref_stride, const uint8_t *mask,
+ int mask_stride, int invert_mask) {
+ if (width < 16) {
+ aom_highbd_comp_mask_pred_avx2(comp_pred8, pred8, width, height, ref8,
+ ref_stride, mask, mask_stride, invert_mask);
+ return;
+ }
+ constexpr hn::ScalableTag<int16_t> di16;
+ constexpr hn::Rebind<uint8_t, decltype(di16)> du8_half;
+ constexpr int kLanes16 = hn::MaxLanes(di16);
+
+ const uint16_t *pred = CONVERT_TO_SHORTPTR(pred8);
+ const uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
+ int16_t *comp_pred =
+ reinterpret_cast<int16_t *>(CONVERT_TO_SHORTPTR(comp_pred8));
+ const int16_t *src0 =
+ reinterpret_cast<const int16_t *>(invert_mask ? pred : ref);
+ const int16_t *src1 =
+ reinterpret_cast<const int16_t *>(invert_mask ? ref : pred);
+ const int stride0 = invert_mask ? width : ref_stride;
+ const int stride1 = invert_mask ? ref_stride : width;
+
+ if (width == 16) {
+ constexpr int kRows = (16 < kLanes16) ? (kLanes16 / 16) : 1;
+ for (int i = 0; i < height; i += kRows) {
+ const auto s0 = LoadRowsU<kRows>(di16, src0, stride0);
+ const auto s1 = LoadRowsU<kRows>(di16, src1, stride1);
+ const auto m8 = LoadRowsU<kRows>(du8_half, mask, mask_stride);
+ const auto m16 = hn::PromoteTo(di16, m8);
+ const auto comp = HighbdCompMaskPredLine(di16, s0, s1, m16);
+ hn::StoreU(comp, di16, comp_pred);
+ src0 += stride0 * kRows;
+ src1 += stride1 * kRows;
+ mask += mask_stride * kRows;
+ comp_pred += 16 * kRows;
+ }
+ } else {
+ for (int i = 0; i < height; ++i) {
+ for (int x = 0; x < width; x += kLanes16) {
+ const auto s0 = hn::LoadU(di16, src0 + x);
+ const auto s1 = hn::LoadU(di16, src1 + x);
+ const auto m8 = hn::LoadU(du8_half, mask + x);
+ const auto m16 = hn::PromoteTo(di16, m8);
+ const auto comp = HighbdCompMaskPredLine(di16, s0, s1, m16);
+ hn::StoreU(comp, di16, comp_pred);
+ comp_pred += kLanes16;
+ }
+ src0 += stride0;
+ src1 += stride1;
+ mask += mask_stride;
+ }
+ }
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
} // namespace HWY_NAMESPACE
} // namespace
@@ -857,6 +1134,84 @@ HWY_ATTR HWY_INLINE unsigned int SubPixelVarianceWxH(
X(64, 32, 32, 6, 5, suffix) \
FOR_EACH_SUBPIX_VAR_64_128_RECT(X, suffix)
+#if !CONFIG_REALTIME_ONLY
+#define FOBMC_VAR(W, H, suffix) \
+ extern "C" unsigned int aom_obmc_variance##W##x##H##_##suffix( \
+ const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
+ const int32_t *mask, unsigned int *sse); \
+ HWY_ATTR unsigned int aom_obmc_variance##W##x##H##_##suffix( \
+ const uint8_t *pre, int pre_stride, const int32_t *wsrc, \
+ const int32_t *mask, unsigned int *sse) { \
+ static_assert((W) >= 16, "FOBMC_VAR requires W >= 16."); \
+ int sum = 0; \
+ HWY_NAMESPACE::ObmcVarianceW16N(pre, pre_stride, wsrc, mask, sse, &sum, W, \
+ H); \
+ return *sse - static_cast<unsigned int>( \
+ (static_cast<int64_t>(sum) * sum) / ((W) * (H))); \
+ }
+
+#define FOR_EACH_OBMC_VAR_BLOCK_SIZE(X, suffix) \
+ X(128, 128, suffix) \
+ X(128, 64, suffix) \
+ X(64, 128, suffix) \
+ X(64, 64, suffix) \
+ X(64, 32, suffix) \
+ X(64, 16, suffix) \
+ X(32, 64, suffix) \
+ X(32, 32, suffix) \
+ X(32, 16, suffix) \
+ X(32, 8, suffix) \
+ X(16, 64, suffix) \
+ X(16, 32, suffix) \
+ X(16, 16, suffix) \
+ X(16, 8, suffix) \
+ X(16, 4, suffix)
+#else
+#define FOBMC_VAR(W, H, suffix)
+#define FOR_EACH_OBMC_VAR_BLOCK_SIZE(X, suffix)
+#endif // !CONFIG_REALTIME_ONLY
+
+#if CONFIG_AV1_HIGHBITDEPTH
+#define FHIGHBD_COMP_MASK_PRED(suffix) \
+ extern "C" void aom_highbd_comp_mask_pred_##suffix( \
+ uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, \
+ const uint8_t *ref8, int ref_stride, const uint8_t *mask, \
+ int mask_stride, int invert_mask); \
+ HWY_ATTR void aom_highbd_comp_mask_pred_##suffix( \
+ uint8_t *comp_pred8, const uint8_t *pred8, int width, int height, \
+ const uint8_t *ref8, int ref_stride, const uint8_t *mask, \
+ int mask_stride, int invert_mask) { \
+ HWY_NAMESPACE::HighbdCompMaskPred(comp_pred8, pred8, width, height, ref8, \
+ ref_stride, mask, mask_stride, \
+ invert_mask); \
+ }
+#else
+#define FHIGHBD_COMP_MASK_PRED(suffix)
+#endif
+
+#define FCOMP_AVG_MASK_PRED(suffix) \
+ extern "C" void aom_comp_avg_pred_##suffix( \
+ uint8_t *comp_pred, const uint8_t *pred, int width, int height, \
+ const uint8_t *ref, int ref_stride); \
+ HWY_ATTR void aom_comp_avg_pred_##suffix( \
+ uint8_t *comp_pred, const uint8_t *pred, int width, int height, \
+ const uint8_t *ref, int ref_stride) { \
+ HWY_NAMESPACE::CompAvgPred(comp_pred, pred, width, height, ref, \
+ ref_stride); \
+ } \
+ extern "C" void aom_comp_mask_pred_##suffix( \
+ uint8_t *comp_pred, const uint8_t *pred, int width, int height, \
+ const uint8_t *ref, int ref_stride, const uint8_t *mask, \
+ int mask_stride, int invert_mask); \
+ HWY_ATTR void aom_comp_mask_pred_##suffix( \
+ uint8_t *comp_pred, const uint8_t *pred, int width, int height, \
+ const uint8_t *ref, int ref_stride, const uint8_t *mask, \
+ int mask_stride, int invert_mask) { \
+ HWY_NAMESPACE::CompMaskPred(comp_pred, pred, width, height, ref, \
+ ref_stride, mask, mask_stride, invert_mask); \
+ } \
+ FHIGHBD_COMP_MASK_PRED(suffix)
+
HWY_AFTER_NAMESPACE();
#endif // AOM_AOM_DSP_VARIANCE_HWY_H_
diff --git a/aom_dsp/x86/variance_hwy_avx512.cc b/aom_dsp/x86/variance_hwy_avx512.cc
index d9ab225131..fd88856721 100644
--- a/aom_dsp/x86/variance_hwy_avx512.cc
+++ b/aom_dsp/x86/variance_hwy_avx512.cc
@@ -21,3 +21,6 @@ FCALC_VAR_STAT(avx512)
FOR_EACH_SUBPIX_VAR_32XH(FSUBPIX_VAR_32XH, avx512)
FOR_EACH_SUBPIX_VAR_64_128(FSUBPIX_VAR_64_128, avx512)
+
+FOR_EACH_OBMC_VAR_BLOCK_SIZE(FOBMC_VAR, avx512)
+FCOMP_AVG_MASK_PRED(avx512)
diff --git a/test/comp_mask_pred_test.cc b/test/comp_mask_pred_test.cc
index be6a1fbefe..0f1b529471 100644
--- a/test/comp_mask_pred_test.cc
+++ b/test/comp_mask_pred_test.cc
@@ -18,6 +18,7 @@
#include "aom/aom_codec.h"
#include "aom/aom_integer.h"
+#include "aom_dsp/blend.h"
#include "aom_dsp/variance.h"
#include "aom_mem/aom_mem.h"
#include "aom_ports/aom_timer.h"
@@ -39,10 +40,12 @@ using comp_avg_pred_func = void (*)(uint8_t *comp_pred, const uint8_t *pred,
int width, int height, const uint8_t *ref,
int ref_stride);
-#if HAVE_SSSE3 || HAVE_SSE2 || HAVE_AVX2 || HAVE_NEON
+#if HAVE_SSSE3 || HAVE_SSE2 || HAVE_AVX2 || HAVE_AVX512 || HAVE_NEON
const BLOCK_SIZE kCompMaskPredParams[] = {
- BLOCK_8X8, BLOCK_8X16, BLOCK_8X32, BLOCK_16X8, BLOCK_16X16,
- BLOCK_16X32, BLOCK_32X8, BLOCK_32X16, BLOCK_32X32
+ BLOCK_8X8, BLOCK_8X16, BLOCK_8X32, BLOCK_16X8, BLOCK_16X16,
+ BLOCK_16X32, BLOCK_16X64, BLOCK_32X8, BLOCK_32X16, BLOCK_32X32,
+ BLOCK_32X64, BLOCK_64X16, BLOCK_64X32, BLOCK_64X64, BLOCK_64X128,
+ BLOCK_128X64, BLOCK_128X128
};
#endif
@@ -128,15 +131,27 @@ void AV1CompMaskPredTest::RunCheckOutput(comp_mask_pred_func test_impl,
const int w = block_size_wide[bsize];
const int h = block_size_high[bsize];
const int wedge_types = get_wedge_types_lookup(bsize);
- for (int wedge_index = 0; wedge_index < wedge_types; ++wedge_index) {
- const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize);
+ if (wedge_types > 0) {
+ for (int wedge_index = 0; wedge_index < wedge_types; ++wedge_index) {
+ const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize);
+ aom_comp_mask_pred_c(comp_pred1_, pred_, w, h, ref_, MAX_SB_SIZE, mask, w,
+ inv);
+ test_impl(comp_pred2_, pred_, w, h, ref_, MAX_SB_SIZE, mask, w, inv);
+
+ ASSERT_EQ(CheckResult(w, h), true)
+ << " wedge " << wedge_index << " inv " << inv;
+ }
+ } else {
+ DECLARE_ALIGNED(16, uint8_t, mask[MAX_SB_SQUARE]);
+ for (int i = 0; i < w * h; ++i) {
+ mask[i] = rnd_(AOM_BLEND_A64_MAX_ALPHA + 1);
+ }
aom_comp_mask_pred_c(comp_pred1_, pred_, w, h, ref_, MAX_SB_SIZE, mask, w,
inv);
test_impl(comp_pred2_, pred_, w, h, ref_, MAX_SB_SIZE, mask, w, inv);
- ASSERT_EQ(CheckResult(w, h), true)
- << " wedge " << wedge_index << " inv " << inv;
+ ASSERT_EQ(CheckResult(w, h), true) << " random mask inv " << inv;
}
}
@@ -145,8 +160,16 @@ void AV1CompMaskPredTest::RunSpeedTest(comp_mask_pred_func test_impl,
const int w = block_size_wide[bsize];
const int h = block_size_high[bsize];
const int wedge_types = get_wedge_types_lookup(bsize);
- int wedge_index = wedge_types / 2;
- const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize);
+ DECLARE_ALIGNED(16, uint8_t, rand_mask[MAX_SB_SQUARE]);
+ const uint8_t *mask = nullptr;
+ if (wedge_types > 0) {
+ mask = av1_get_contiguous_soft_mask(wedge_types / 2, 1, bsize);
+ } else {
+ for (int i = 0; i < w * h; ++i) {
+ rand_mask[i] = rnd_(AOM_BLEND_A64_MAX_ALPHA + 1);
+ }
+ mask = rand_mask;
+ }
const int num_loops = 1000000000 / (w + h);
comp_mask_pred_func funcs[2] = { aom_comp_mask_pred_c, test_impl };
@@ -193,6 +216,13 @@ INSTANTIATE_TEST_SUITE_P(
::testing::ValuesIn(kCompMaskPredParams)));
#endif
+#if CONFIG_HIGHWAY && HAVE_AVX512
+INSTANTIATE_TEST_SUITE_P(
+ AVX512, AV1CompMaskPredTest,
+ ::testing::Combine(::testing::Values(&aom_comp_mask_pred_avx512),
+ ::testing::ValuesIn(kCompMaskPredParams)));
+#endif
+
#if HAVE_NEON
INSTANTIATE_TEST_SUITE_P(
NEON, AV1CompMaskPredTest,
@@ -200,7 +230,7 @@ INSTANTIATE_TEST_SUITE_P(
::testing::ValuesIn(kCompMaskPredParams)));
#endif
-#if HAVE_SSSE3 || HAVE_SSE2 || HAVE_AVX2 || HAVE_NEON
+#if HAVE_SSSE3 || HAVE_SSE2 || HAVE_AVX2 || HAVE_AVX512 || HAVE_NEON
const BLOCK_SIZE kValidBlockSize[] = {
BLOCK_4X4, BLOCK_8X8, BLOCK_8X16, BLOCK_8X32, BLOCK_16X8,
BLOCK_16X16, BLOCK_16X32, BLOCK_32X8, BLOCK_32X16, BLOCK_32X32,
@@ -413,6 +443,13 @@ INSTANTIATE_TEST_SUITE_P(
::testing::ValuesIn(kValidBlockSize)));
#endif
+#if CONFIG_HIGHWAY && HAVE_AVX512
+INSTANTIATE_TEST_SUITE_P(
+ AVX512, AV1CompAvgPredTest,
+ ::testing::Combine(::testing::Values(&aom_comp_avg_pred_avx512),
+ ::testing::ValuesIn(kValidBlockSize)));
+#endif
+
#if HAVE_NEON
INSTANTIATE_TEST_SUITE_P(
NEON, AV1CompAvgPredTest,
@@ -520,9 +557,25 @@ void AV1HighbdCompMaskPredTest::RunCheckOutput(
ref_buffer_[i] = rnd_.Rand16() & ((1 << bd_) - 1);
}
- for (int wedge_index = 0; wedge_index < wedge_types; ++wedge_index) {
- const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize);
+ if (wedge_types > 0) {
+ for (int wedge_index = 0; wedge_index < wedge_types; ++wedge_index) {
+ const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize);
+
+ aom_highbd_comp_mask_pred_c(
+ CONVERT_TO_BYTEPTR(comp_pred1_), CONVERT_TO_BYTEPTR(pred_), w, h,
+ CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE, mask, w, inv);
+ test_impl(CONVERT_TO_BYTEPTR(comp_pred2_), CONVERT_TO_BYTEPTR(pred_), w,
+ h, CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE, mask, w, inv);
+
+ ASSERT_EQ(CheckResult(w, h), true)
+ << " wedge " << wedge_index << " inv " << inv;
+ }
+ } else {
+ DECLARE_ALIGNED(16, uint8_t, mask[MAX_SB_SQUARE]);
+ for (int i = 0; i < w * h; ++i) {
+ mask[i] = rnd_(AOM_BLEND_A64_MAX_ALPHA + 1);
+ }
aom_highbd_comp_mask_pred_c(
CONVERT_TO_BYTEPTR(comp_pred1_), CONVERT_TO_BYTEPTR(pred_), w, h,
CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE, mask, w, inv);
@@ -530,8 +583,7 @@ void AV1HighbdCompMaskPredTest::RunCheckOutput(
test_impl(CONVERT_TO_BYTEPTR(comp_pred2_), CONVERT_TO_BYTEPTR(pred_), w, h,
CONVERT_TO_BYTEPTR(ref_), MAX_SB_SIZE, mask, w, inv);
- ASSERT_EQ(CheckResult(w, h), true)
- << " wedge " << wedge_index << " inv " << inv;
+ ASSERT_EQ(CheckResult(w, h), true) << " random mask inv " << inv;
}
}
@@ -542,7 +594,6 @@ void AV1HighbdCompMaskPredTest::RunSpeedTest(
const int w = block_size_wide[bsize];
const int h = block_size_high[bsize];
const int wedge_types = get_wedge_types_lookup(bsize);
- int wedge_index = wedge_types / 2;
for (int i = 0; i < MAX_SB_SQUARE; ++i) {
pred_[i] = rnd_.Rand16() & ((1 << bd_) - 1);
@@ -551,7 +602,16 @@ void AV1HighbdCompMaskPredTest::RunSpeedTest(
ref_buffer_[i] = rnd_.Rand16() & ((1 << bd_) - 1);
}
- const uint8_t *mask = av1_get_contiguous_soft_mask(wedge_index, 1, bsize);
+ DECLARE_ALIGNED(16, uint8_t, rand_mask[MAX_SB_SQUARE]);
+ const uint8_t *mask = nullptr;
+ if (wedge_types > 0) {
+ mask = av1_get_contiguous_soft_mask(wedge_types / 2, 1, bsize);
+ } else {
+ for (int i = 0; i < w * h; ++i) {
+ rand_mask[i] = rnd_(AOM_BLEND_A64_MAX_ALPHA + 1);
+ }
+ mask = rand_mask;
+ }
const int num_loops = 1000000000 / (w + h);
highbd_comp_mask_pred_func funcs[2] = { aom_highbd_comp_mask_pred_c,
@@ -602,6 +662,14 @@ INSTANTIATE_TEST_SUITE_P(
::testing::Range(8, 13, 2)));
#endif
+#if CONFIG_HIGHWAY && HAVE_AVX512
+INSTANTIATE_TEST_SUITE_P(
+ AVX512, AV1HighbdCompMaskPredTest,
+ ::testing::Combine(::testing::Values(&aom_highbd_comp_mask_pred_avx512),
+ ::testing::ValuesIn(kCompMaskPredParams),
+ ::testing::Range(8, 13, 2)));
+#endif
+
#if HAVE_SSE2
INSTANTIATE_TEST_SUITE_P(
SSE2, AV1HighbdCompMaskPredTest,
diff --git a/test/obmc_variance_test.cc b/test/obmc_variance_test.cc
index 16482f70cc..23d7fa5916 100644
--- a/test/obmc_variance_test.cc
+++ b/test/obmc_variance_test.cc
@@ -194,6 +194,30 @@ INSTANTIATE_TEST_SUITE_P(AVX2, ObmcVarianceTest,
::testing::ValuesIn(avx2_functions));
#endif // HAVE_AVX2
+#if CONFIG_HIGHWAY && HAVE_AVX512
+const ObmcVarianceTest::ParamType avx512_functions[] = {
+ TestFuncs(aom_obmc_variance128x128_c, aom_obmc_variance128x128_avx512),
+ TestFuncs(aom_obmc_variance128x64_c, aom_obmc_variance128x64_avx512),
+ TestFuncs(aom_obmc_variance64x128_c, aom_obmc_variance64x128_avx512),
+ TestFuncs(aom_obmc_variance64x64_c, aom_obmc_variance64x64_avx512),
+ TestFuncs(aom_obmc_variance64x32_c, aom_obmc_variance64x32_avx512),
+ TestFuncs(aom_obmc_variance32x64_c, aom_obmc_variance32x64_avx512),
+ TestFuncs(aom_obmc_variance32x32_c, aom_obmc_variance32x32_avx512),
+ TestFuncs(aom_obmc_variance32x16_c, aom_obmc_variance32x16_avx512),
+ TestFuncs(aom_obmc_variance16x32_c, aom_obmc_variance16x32_avx512),
+ TestFuncs(aom_obmc_variance16x16_c, aom_obmc_variance16x16_avx512),
+ TestFuncs(aom_obmc_variance16x8_c, aom_obmc_variance16x8_avx512),
+
+ TestFuncs(aom_obmc_variance64x16_c, aom_obmc_variance64x16_avx512),
+ TestFuncs(aom_obmc_variance16x64_c, aom_obmc_variance16x64_avx512),
+ TestFuncs(aom_obmc_variance32x8_c, aom_obmc_variance32x8_avx512),
+ TestFuncs(aom_obmc_variance16x4_c, aom_obmc_variance16x4_avx512),
+};
+
+INSTANTIATE_TEST_SUITE_P(AVX512, ObmcVarianceTest,
+ ::testing::ValuesIn(avx512_functions));
+#endif // CONFIG_HIGHWAY && HAVE_AVX512
+
#if HAVE_NEON
const ObmcVarianceTest::ParamType neon_functions[] = {
TestFuncs(aom_obmc_variance128x128_c, aom_obmc_variance128x128_neon),