Commit 19ad10ac60 for aom
commit 19ad10ac60a84f9f7b1b478268046e25900a89e4
Author: Ihor Buhaiov <freedbrt@gmail.com>
Date: Thu Oct 15 22:31:41 2026 +0200
Pad the rows of the IntraBC frame hash buffers
The IntraBC table fill reads the frame hash buffers column by column,
moving down 2, 4 or 8 rows at a time. The rows were exactly the frame
width apart. When the width is a multiple of 128 values, like 1280, 1920
and 2560, every step is a multiple of 1 KB. With 64 byte lines the reads
of a column then land in at most 1/16 of the cache sets. A column
doesn't fit in that part of the L2, so its lines are evicted before the
next column reuses them. Other widths are hit less or not at all.
Make the row stride an odd multiple of 8 values, the width rounded up
to 8 with bit 3 set. A step of 2 rows is then an odd number of cache
lines and a column spreads over all the sets. Steps of 4 and 8 rows use
half and a quarter of them, which is enough for the shorter columns of
those passes. The padding is at most 15 values per row.
The hashes and the tables are the same, so the output does not change.
Packet hashes match before and after for RT, good quality and all intra
at 1 and 4 threads, on Arm and on x86.
av1_generate_block_hash_value() used the picture only for its width, so
it takes the stride instead.
On five 2560x1440 screen recordings at speed 11 with IntraBC on (one is
2559 wide), 20 key frames each, key frames use 2.0% to 5.6% less CPU at
1 thread on an AMD Ryzen 9 5900X with MSVC, and the wall time at 4
threads drops by 1.5% to 4.5%. On an Apple M3 Pro they use 0.1% to 0.6%
less CPU at 1 thread, and the wall time at 4 threads drops by up to 1.4%
(the 2559 wide one is 0.2% slower). Measured with a timing patch on the
5900X, the table fill alone takes 12.4 ms instead of 22.8 ms per key
frame on one of the recordings.
Bug: 576320193
Change-Id: I9ddc1031a1a3b484c9ffec46adcd6c24e73f52f1
diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index c7eb82a7ff..0fd9691206 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -2073,6 +2073,17 @@ static void free_block_hash_buffers(IntraBCHashInfo *intrabc_hash_info) {
intrabc_hash_info->frame_hash_height = 0;
}
+// Returns the distance in values between rows of the frame hash buffers. The
+// table fill reads the buffers column by column, 2 or more rows at a time.
+// When the width has a large power of 2 factor, the reads of a column share
+// their low address bits and map to only a few cache sets. The lines a column
+// loads are then evicted before the next column reads them again. With an odd
+// multiple of 8 values, a step of 2 rows is an odd number of 64 byte cache
+// lines, so a column spreads over all the cache sets.
+static int get_intrabc_hash_stride(int width) {
+ return ALIGN_POWER_OF_TWO(width, 3) | 8;
+}
+
// Gets the region covered by the hash table with the given index. That is the
// region of the tile with this index, or the whole frame if there is one table.
static void get_intrabc_hash_region(const YV12_BUFFER_CONFIG *source,
@@ -2105,11 +2116,13 @@ bool av1_build_intrabc_hash_table(AV1_COMP *cpi, int table_idx) {
pic_height != intrabc_hash_info->frame_hash_height) {
return false;
}
+ const int hash_stride = get_intrabc_hash_stride(pic_width);
IntraBCHashRegion region;
get_intrabc_hash_region(cpi->source, cm, intrabc_hash_info->num_tile_tables,
table_idx, ®ion);
- av1_generate_block_2x2_hash_value(cpi->source, ®ion, block_hash_values[0]);
+ av1_generate_block_2x2_hash_value(cpi->source, ®ion, block_hash_values[0],
+ hash_stride);
// Hash data generated for screen contents is used for intraBC ME
const int min_alloc_size = block_size_wide[cm->mi_params.mi_alloc_bsize];
int max_sb_size = 1 << (cm->seq_params->mib_size_log2 + MI_SIZE_LOG2);
@@ -2121,13 +2134,13 @@ bool av1_build_intrabc_hash_table(AV1_COMP *cpi, int table_idx) {
int src_idx = 0;
for (int size = 4; size <= max_sb_size; size *= 2, src_idx = !src_idx) {
const int dst_idx = !src_idx;
- av1_generate_block_hash_value(intrabc_hash_info, cpi->source, ®ion, size,
+ av1_generate_block_hash_value(intrabc_hash_info, ®ion, size,
block_hash_values[src_idx],
- block_hash_values[dst_idx]);
+ block_hash_values[dst_idx], hash_stride);
if (size >= min_alloc_size &&
!av1_add_to_hash_map_by_row_with_precal_data(
&intrabc_hash_info->tile_tables[table_idx],
- block_hash_values[dst_idx], pic_width, ®ion, size)) {
+ block_hash_values[dst_idx], hash_stride, ®ion, size)) {
return false;
}
}
@@ -2146,6 +2159,7 @@ static void build_intrabc_hash_tables(AV1_COMP *cpi) {
IntraBCHashInfo *const intrabc_hash_info = &cpi->td.mb.intrabc_hash_info;
const int pic_width = cpi->source->y_crop_width;
const int pic_height = cpi->source->y_crop_height;
+ const int hash_stride = get_intrabc_hash_stride(pic_width);
const int num_tiles = cm->tiles.cols * cm->tiles.rows;
const int num_tables = num_tiles <= MAX_INTRABC_HASH_TABLES ? num_tiles : 1;
bool error = false;
@@ -2153,7 +2167,7 @@ static void build_intrabc_hash_tables(AV1_COMP *cpi) {
free_block_hash_buffers(intrabc_hash_info);
for (int j = 0; j < 2; ++j) {
intrabc_hash_info->frame_hash_buffer[j] = (uint32_t *)aom_malloc(
- sizeof(*intrabc_hash_info->frame_hash_buffer[j]) * pic_width *
+ sizeof(*intrabc_hash_info->frame_hash_buffer[j]) * hash_stride *
pic_height);
if (!intrabc_hash_info->frame_hash_buffer[j]) {
error = true;
diff --git a/av1/encoder/hash_motion.c b/av1/encoder/hash_motion.c
index c1ffdb7ed9..74f410951f 100644
--- a/av1/encoder/hash_motion.c
+++ b/av1/encoder/hash_motion.c
@@ -206,17 +206,17 @@ const block_hash *av1_hash_get_first_block_hash(const hash_table *p_hash_table,
void av1_generate_block_2x2_hash_value(const YV12_BUFFER_CONFIG *picture,
const IntraBCHashRegion *region,
- uint32_t *pic_block_hash) {
+ uint32_t *pic_block_hash,
+ int pic_block_hash_stride) {
const int width = 2;
const int height = 2;
- const int pic_width = picture->y_crop_width;
const int x_end = region->x_end - width + 1;
const int y_end = region->y_end - height + 1;
if (picture->flags & YV12_FLAG_HIGHBITDEPTH) {
uint16_t p[4];
for (int y_pos = region->y_start; y_pos < y_end; y_pos++) {
- uint32_t *const row_hash = pic_block_hash + y_pos * pic_width;
+ uint32_t *const row_hash = pic_block_hash + y_pos * pic_block_hash_stride;
for (int x_pos = region->x_start; x_pos < x_end; x_pos++) {
get_pixels_in_1D_short_array_by_block_2x2(
CONVERT_TO_SHORTPTR(picture->y_buffer) + y_pos * picture->y_stride +
@@ -231,7 +231,7 @@ void av1_generate_block_2x2_hash_value(const YV12_BUFFER_CONFIG *picture,
} else {
uint8_t p[4];
for (int y_pos = region->y_start; y_pos < y_end; y_pos++) {
- uint32_t *const row_hash = pic_block_hash + y_pos * pic_width;
+ uint32_t *const row_hash = pic_block_hash + y_pos * pic_block_hash_stride;
for (int x_pos = region->x_start; x_pos < x_end; x_pos++) {
get_pixels_in_1D_char_array_by_block_2x2(
picture->y_buffer + y_pos * picture->y_stride + x_pos,
@@ -247,14 +247,13 @@ void av1_generate_block_2x2_hash_value(const YV12_BUFFER_CONFIG *picture,
}
void av1_generate_block_hash_value(IntraBCHashInfo *intrabc_hash_info,
- const YV12_BUFFER_CONFIG *picture,
const IntraBCHashRegion *region,
int block_size,
const uint32_t *src_pic_block_hash,
- uint32_t *dst_pic_block_hash) {
+ uint32_t *dst_pic_block_hash,
+ int pic_block_hash_stride) {
CRC32C *calc = &intrabc_hash_info->crc_calculator;
- const int pic_width = picture->y_crop_width;
const int x_end = region->x_end - block_size + 1;
const int y_end = region->y_end - block_size + 1;
const int src_size = block_size >> 1;
@@ -263,7 +262,7 @@ void av1_generate_block_hash_value(IntraBCHashInfo *intrabc_hash_info,
const int length = sizeof(p);
for (int y_pos = region->y_start; y_pos < y_end; y_pos++) {
- int pos = y_pos * pic_width + region->x_start;
+ int pos = y_pos * pic_block_hash_stride + region->x_start;
for (int x_pos = region->x_start; x_pos < x_end; x_pos++) {
// Build up a bigger block from 4 smaller, non-overlapping source block
// hashes, and compute its hash. Note: source blocks at the right and
@@ -271,8 +270,9 @@ void av1_generate_block_hash_value(IntraBCHashInfo *intrabc_hash_info,
// considered into the block hash value generation process.
p[0] = src_pic_block_hash[pos];
p[1] = src_pic_block_hash[pos + src_size];
- p[2] = src_pic_block_hash[pos + src_size * pic_width];
- p[3] = src_pic_block_hash[pos + src_size * pic_width + src_size];
+ p[2] = src_pic_block_hash[pos + src_size * pic_block_hash_stride];
+ p[3] =
+ src_pic_block_hash[pos + src_size * pic_block_hash_stride + src_size];
// TODO: bug aomedia:433531610 - serialize input values in a way that's
// independent of the computer architecture's endianness
dst_pic_block_hash[pos] =
@@ -286,7 +286,7 @@ void av1_generate_block_hash_value(IntraBCHashInfo *intrabc_hash_info,
// counts of that block size must be zero on entry, as av1_hash_tables_create()
// leaves them. On return they hold the number of entries in each bucket.
bool av1_add_to_hash_map_by_row_with_precal_data(
- hash_table *p_hash_table, const uint32_t *pic_hash, int pic_width,
+ hash_table *p_hash_table, const uint32_t *pic_hash, int pic_hash_stride,
const IntraBCHashRegion *region, int block_size) {
const int x_start = region->x_start;
const int y_start = region->y_start;
@@ -316,7 +316,7 @@ bool av1_add_to_hash_map_by_row_with_precal_data(
// added in, so count them first in raster order. Flat areas give long runs
// of the same bucket, so count a whole run at once.
for (int y_pos = y_start; y_pos < y_end; y_pos++) {
- const uint32_t *const row_hash = pic_hash + y_pos * pic_width;
+ const uint32_t *const row_hash = pic_hash + y_pos * pic_hash_stride;
int bucket = row_hash[x_start] & crc_mask;
int run = 0;
for (int x_pos = x_start; x_pos < x_end; x_pos++) {
@@ -377,7 +377,7 @@ bool av1_add_to_hash_map_by_row_with_precal_data(
while (step > 1) {
for (int x_pos = x_start + x_offset; x_pos < x_end; x_pos += step) {
for (int y_pos = y_start + y_offset; y_pos < y_end; y_pos += step) {
- const int pos = y_pos * pic_width + x_pos;
+ const int pos = y_pos * pic_hash_stride + x_pos;
const int bucket = pic_hash[pos] & crc_mask;
const int count = bucket_count[bucket];
assert(count <= kMaxCandidatesPerHashBucket);
diff --git a/av1/encoder/hash_motion.h b/av1/encoder/hash_motion.h
index 03558d2bd5..95e2b51893 100644
--- a/av1/encoder/hash_motion.h
+++ b/av1/encoder/hash_motion.h
@@ -101,19 +101,20 @@ int32_t av1_hash_table_count(const hash_table *p_hash_table,
// Returns the first of the av1_hash_table_count() entries for hash_value.
const block_hash *av1_hash_get_first_block_hash(const hash_table *p_hash_table,
uint32_t hash_value);
-// These fill the hash values of the blocks that lie inside region. The buffers
-// use the width of the picture as stride.
+// These fill the hash values of the blocks that lie inside region. The stride
+// parameters give the distance in values between rows of the hash buffers.
void av1_generate_block_2x2_hash_value(const YV12_BUFFER_CONFIG *picture,
const IntraBCHashRegion *region,
- uint32_t *pic_block_hash);
+ uint32_t *pic_block_hash,
+ int pic_block_hash_stride);
void av1_generate_block_hash_value(IntraBCHashInfo *intra_bc_hash_info,
- const YV12_BUFFER_CONFIG *picture,
const IntraBCHashRegion *region,
int block_size,
const uint32_t *src_pic_block_hash,
- uint32_t *dst_pic_block_hash);
+ uint32_t *dst_pic_block_hash,
+ int pic_block_hash_stride);
bool av1_add_to_hash_map_by_row_with_precal_data(
- hash_table *p_hash_table, const uint32_t *pic_hash, int pic_width,
+ hash_table *p_hash_table, const uint32_t *pic_hash, int pic_hash_stride,
const IntraBCHashRegion *region, int block_size);
// check whether the block starts from (x_start, y_start) with the size of