Commit f469a9e23e for aom
commit f469a9e23ebec4168a645c29a5845ebee9e04e64
Author: Ihor Buhaiov <freedbrt@gmail.com>
Date: Thu Oct 15 20:06:18 2026 +0200
Add CRC kernels for the IntraBC hash passes
On every key frame with IntraBC on, av1_generate_block_hash_value()
hashes each pixel position once per block size. It did that by calling
av1_get_crc32c_value() through RTCD for every 16 byte input, after
copying the 4 input values into a stack array. The call and the copy
cost more than the 2 CRC instructions that do the work.
Move the whole pass into a new RTCD function,
av1_combine_block_hashes(). The SSE4.2 and Arm CRC32 versions keep the
inputs in registers and inline the CRC instructions. The C version is
the old loop. The hashes are the same, so the output does not change.
Tested on seven screen clips, 20 frames each, at speed 11 with IntraBC
on and every frame a key frame. Six are 2560x1440 or 2559x1372
recordings and one is a 1920x1080 CTC clip. Change in CPU time at 1
thread and in wall time at 4 threads with 2x2 tiles, per clip, on an
Apple M3 Pro (clang) and an AMD Ryzen 9 5900X (MSVC):
4:2:0 4:4:4
M3 Pro, 1 thread -5.2% to -7.3% -4.9% to -6.9%
M3 Pro, 4 threads -2.8% to -5.2% -3.0% to -4.8%
5900X, 1 thread -6.6% to -11.4% -7.7% to -9.3%
5900X, 4 threads -4.1% to -7.5% -3.9% to -6.8%
The streams match before and after on both machines.
The new unit test compares each version with the C one for every sub
block size, odd widths and flat input.
Bug: 576320193
Change-Id: If0559605bf0e88f0e7458b433fc676e03875319b
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index b906ba03fd..3981c57579 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -464,6 +464,8 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
# hash
add_proto qw/uint32_t av1_get_crc32c_value/, "void *crc_calculator, const uint8_t *p, size_t length";
specialize qw/av1_get_crc32c_value sse4_2 arm_crc32/;
+ add_proto qw/void av1_combine_block_hashes/, "void *crc_calculator, const uint32_t *src, uint32_t *dst, int stride, int width, int height, int sub_size";
+ specialize qw/av1_combine_block_hashes sse4_2 arm_crc32/;
if (aom_config("CONFIG_REALTIME_ONLY") ne "yes") {
add_proto qw/void av1_compute_stats/, "int wiener_win, const uint8_t *dgd8, const uint8_t *src8, int16_t *dgd_avg, int16_t *src_avg, int h_start, int h_end, int v_start, int v_end, int dgd_stride, int src_stride, int64_t *M, int64_t *H, int use_downsampled_wiener_stats";
diff --git a/av1/encoder/arm/hash_arm_crc32.c b/av1/encoder/arm/hash_arm_crc32.c
index 0712b80154..6b2baec2dd 100644
--- a/av1/encoder/arm/hash_arm_crc32.c
+++ b/av1/encoder/arm/hash_arm_crc32.c
@@ -60,3 +60,23 @@ uint32_t av1_get_crc32c_value_arm_crc32(void *crc_calculator, const uint8_t *p,
return ~crc;
}
+
+void av1_combine_block_hashes_arm_crc32(void *crc_calculator,
+ const uint32_t *src, uint32_t *dst,
+ int stride, int width, int height,
+ int sub_size) {
+ (void)crc_calculator;
+ for (int y = 0; y < height; y++) {
+ const uint32_t *top = src + y * stride;
+ const uint32_t *bottom = top + sub_size * stride;
+ uint32_t *out = dst + y * stride;
+ for (int x = 0; x < width; x++) {
+ // Hash the 4 values from registers, in the same byte order as the C
+ // version reads them from memory.
+ uint32_t crc = 0xFFFFFFFF;
+ crc = __crc32cd(crc, top[x] | (uint64_t)top[x + sub_size] << 32);
+ crc = __crc32cd(crc, bottom[x] | (uint64_t)bottom[x + sub_size] << 32);
+ out[x] = ~crc;
+ }
+ }
+}
diff --git a/av1/encoder/hash.c b/av1/encoder/hash.c
index bcb9147870..6fae8465c7 100644
--- a/av1/encoder/hash.c
+++ b/av1/encoder/hash.c
@@ -12,6 +12,7 @@
#include "av1/encoder/hash.h"
#include <stddef.h>
+#include <string.h>
#include "config/av1_rtcd.h"
@@ -70,3 +71,26 @@ uint32_t av1_get_crc32c_value_c(void *c, const uint8_t *buf, size_t len) {
}
return (uint32_t)crc ^ 0xffffffff;
}
+
+// For each of the width x height positions, hashes the 4 sub-block hashes at
+// that position and at the offsets sub_size right, sub_size down and both.
+// src and dst use the same stride.
+void av1_combine_block_hashes_c(void *crc_calculator, const uint32_t *src,
+ uint32_t *dst, int stride, int width,
+ int height, int sub_size) {
+ for (int y = 0; y < height; y++) {
+ const uint32_t *top = src + y * stride;
+ const uint32_t *bottom = top + sub_size * stride;
+ uint32_t *out = dst + y * stride;
+ for (int x = 0; x < width; x++) {
+ const uint32_t v[4] = { top[x], top[x + sub_size], bottom[x],
+ bottom[x + sub_size] };
+ // Hash a byte copy so the static analyzer can follow the byte reads.
+ // TODO: bug aomedia:433531610 - serialize input values in a way that's
+ // independent of the computer architecture's endianness
+ uint8_t p[sizeof(v)];
+ memcpy(p, v, sizeof(v));
+ out[x] = av1_get_crc32c_value_c(crc_calculator, p, sizeof(p));
+ }
+ }
+}
diff --git a/av1/encoder/hash_motion.c b/av1/encoder/hash_motion.c
index 74f410951f..eaf9633e8f 100644
--- a/av1/encoder/hash_motion.c
+++ b/av1/encoder/hash_motion.c
@@ -252,34 +252,19 @@ void av1_generate_block_hash_value(IntraBCHashInfo *intrabc_hash_info,
const uint32_t *src_pic_block_hash,
uint32_t *dst_pic_block_hash,
int pic_block_hash_stride) {
- CRC32C *calc = &intrabc_hash_info->crc_calculator;
-
const int x_end = region->x_end - block_size + 1;
const int y_end = region->y_end - block_size + 1;
- const int src_size = block_size >> 1;
-
- uint32_t p[4];
- const int length = sizeof(p);
-
- for (int y_pos = region->y_start; y_pos < y_end; y_pos++) {
- int pos = y_pos * pic_block_hash_stride + region->x_start;
- for (int x_pos = region->x_start; x_pos < x_end; x_pos++) {
- // Build up a bigger block from 4 smaller, non-overlapping source block
- // hashes, and compute its hash. Note: source blocks at the right and
- // bottom borders cannot be part of larger blocks, therefore they won't be
- // considered into the block hash value generation process.
- p[0] = src_pic_block_hash[pos];
- p[1] = src_pic_block_hash[pos + src_size];
- p[2] = src_pic_block_hash[pos + src_size * pic_block_hash_stride];
- p[3] =
- src_pic_block_hash[pos + src_size * pic_block_hash_stride + src_size];
- // TODO: bug aomedia:433531610 - serialize input values in a way that's
- // independent of the computer architecture's endianness
- dst_pic_block_hash[pos] =
- av1_get_crc32c_value(calc, (uint8_t *)p, length);
- pos++;
- }
- }
+ if (x_end <= region->x_start || y_end <= region->y_start) return;
+ const int offset = region->y_start * pic_block_hash_stride + region->x_start;
+
+ // Build up a bigger block from 4 smaller, non-overlapping source block
+ // hashes, and compute its hash. Note: source blocks at the right and
+ // bottom borders cannot be part of larger blocks, therefore they won't be
+ // considered into the block hash value generation process.
+ av1_combine_block_hashes(
+ &intrabc_hash_info->crc_calculator, src_pic_block_hash + offset,
+ dst_pic_block_hash + offset, pic_block_hash_stride,
+ x_end - region->x_start, y_end - region->y_start, block_size >> 1);
}
// Adds the blocks of one block size that lie inside region to the table. The
diff --git a/av1/encoder/x86/hash_sse42.c b/av1/encoder/x86/hash_sse42.c
index 3c618cc3c5..53df63a567 100644
--- a/av1/encoder/x86/hash_sse42.c
+++ b/av1/encoder/x86/hash_sse42.c
@@ -12,6 +12,7 @@
#include <stdint.h>
#include <smmintrin.h>
+#include "config/aom_config.h"
#include "config/av1_rtcd.h"
// Byte-boundary alignment issues
@@ -51,3 +52,32 @@ uint32_t av1_get_crc32c_value_sse4_2(void *crc_calculator, const uint8_t *p,
CALC_CRC(_mm_crc32_u8, crc, uint8_t, buf, len)
return (crc ^ 0xFFFFFFFF);
}
+
+void av1_combine_block_hashes_sse4_2(void *crc_calculator, const uint32_t *src,
+ uint32_t *dst, int stride, int width,
+ int height, int sub_size) {
+ (void)crc_calculator;
+ for (int y = 0; y < height; y++) {
+ const uint32_t *top = src + y * stride;
+ const uint32_t *bottom = top + sub_size * stride;
+ uint32_t *out = dst + y * stride;
+ for (int x = 0; x < width; x++) {
+ // Hash the 4 values from registers, in the same byte order as the C
+ // version reads them from memory.
+#if AOM_ARCH_X86_64
+ uint64_t crc = 0xFFFFFFFF;
+ crc = _mm_crc32_u64(crc, top[x] | (uint64_t)top[x + sub_size] << 32);
+ crc =
+ _mm_crc32_u64(crc, bottom[x] | (uint64_t)bottom[x + sub_size] << 32);
+ out[x] = (uint32_t)crc ^ 0xFFFFFFFF;
+#else
+ uint32_t crc = 0xFFFFFFFF;
+ crc = _mm_crc32_u32(crc, top[x]);
+ crc = _mm_crc32_u32(crc, top[x + sub_size]);
+ crc = _mm_crc32_u32(crc, bottom[x]);
+ crc = _mm_crc32_u32(crc, bottom[x + sub_size]);
+ out[x] = crc ^ 0xFFFFFFFF;
+#endif
+ }
+ }
+}
diff --git a/test/hash_test.cc b/test/hash_test.cc
index adc072e133..e481203145 100644
--- a/test/hash_test.cc
+++ b/test/hash_test.cc
@@ -12,6 +12,7 @@
#include <cstdlib>
#include <new>
#include <tuple>
+#include <vector>
#include "config/aom_config.h"
#include "config/av1_rtcd.h"
@@ -138,4 +139,100 @@ INSTANTIATE_TEST_SUITE_P(
::testing::ValuesIn(kValidBlockSize)));
#endif
+using combine_block_hashes_func = void (*)(void *crc_calculator,
+ const uint32_t *src, uint32_t *dst,
+ int stride, int width, int height,
+ int sub_size);
+
+class AV1CombineBlockHashesTest
+ : public ::testing::TestWithParam<combine_block_hashes_func> {
+ protected:
+ void SetUp() override {
+ rnd_.Reset(libaom_test::ACMRandom::DeterministicSeed());
+ av1_crc32c_calculator_init(&calc_);
+ }
+
+ uint32_t Rand32() {
+ return static_cast<uint32_t>(rnd_.Rand16()) << 16 | rnd_.Rand16();
+ }
+
+ // Compares the whole output buffer with the C version, so writes outside
+ // the width x height area show up too. flat fills src with few values, like
+ // a screen with large flat areas.
+ void RunCheckOutput(int width, int height, int sub_size, bool flat) {
+ const int stride = width + 2 * sub_size + 3;
+ const int rows = height + 2 * sub_size;
+ std::vector<uint32_t> src(stride * rows);
+ for (uint32_t &v : src) {
+ v = flat ? rnd_.Rand8() % 2 : Rand32();
+ }
+ std::vector<uint32_t> ref(stride * rows, 0xDEADBEEF);
+ std::vector<uint32_t> out(stride * rows, 0xDEADBEEF);
+ av1_combine_block_hashes_c(&calc_, src.data(), ref.data(), stride, width,
+ height, sub_size);
+ GetParam()(&calc_, src.data(), out.data(), stride, width, height, sub_size);
+ ASSERT_EQ(ref, out) << "width " << width << " height " << height
+ << " sub_size " << sub_size << " flat " << flat;
+ }
+
+ libaom_test::ACMRandom rnd_;
+ CRC32C calc_;
+};
+
+TEST_P(AV1CombineBlockHashesTest, CheckOutput) {
+ const int kSubSizes[] = { 2, 4, 8, 16, 32, 64 };
+ const int kWidths[] = { 0, 1, 7, 64, 129 };
+ const int kHeights[] = { 1, 5, 33 };
+ for (int sub_size : kSubSizes) {
+ for (int width : kWidths) {
+ for (int height : kHeights) {
+ RunCheckOutput(width, height, sub_size, false);
+ RunCheckOutput(width, height, sub_size, true);
+ }
+ }
+ }
+}
+
+TEST_P(AV1CombineBlockHashesTest, DISABLED_Speed) {
+ // One 4x4 pass over a 2560x1440 frame.
+ const int stride = 2560;
+ const int height = 1440;
+ const int sub_size = 2;
+ std::vector<uint32_t> src(stride * height);
+ for (uint32_t &v : src) v = Rand32();
+ std::vector<uint32_t> dst(stride * height);
+ const combine_block_hashes_func impls[] = { av1_combine_block_hashes_c,
+ GetParam() };
+ const int width = stride - 2 * sub_size + 1;
+ const int rows = height - 2 * sub_size + 1;
+ const int repeat = 20;
+
+ double time[2];
+ for (int i = 0; i < 2; ++i) {
+ aom_usec_timer timer;
+ aom_usec_timer_start(&timer);
+ for (int j = 0; j < repeat; ++j) {
+ impls[i](&calc_, src.data(), dst.data(), stride, width, rows, sub_size);
+ }
+ aom_usec_timer_mark(&timer);
+ time[i] = static_cast<double>(aom_usec_timer_elapsed(&timer));
+ }
+ printf("combine 2560x1440: %8.2f/%8.2fus (%3.2f)\n", time[0], time[1],
+ time[0] / time[1]);
+}
+
+INSTANTIATE_TEST_SUITE_P(C, AV1CombineBlockHashesTest,
+ ::testing::Values(&av1_combine_block_hashes_c));
+
+#if HAVE_SSE4_2
+INSTANTIATE_TEST_SUITE_P(SSE4_2, AV1CombineBlockHashesTest,
+ ::testing::Values(&av1_combine_block_hashes_sse4_2));
+#endif
+
+#if HAVE_ARM_CRC32
+INSTANTIATE_TEST_SUITE_P(
+ ARM_CRC32, AV1CombineBlockHashesTest,
+ ::testing::Values(&av1_combine_block_hashes_arm_crc32));
+#endif
+
} // namespace