Commit 760bdf95d7 for aom
commit 760bdf95d7d36d98159247c1b81fa85d7263dcfe
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Oct 31 09:54:04 2026 -0700
av1: Optimize reconintra setup and tx_search
Bypass full neighbor buffer initialization and top-left corner setup for
pure vertical and horizontal directional predictors, compute top-right
and bottom-left availability on demand, populate txk_map directly from
set bits of allowed_tx_mask when unordered, and add a single-block fast
path in av1_txfm_rd_in_plane.
Benchmarking results (incremental vs prev CL / cumulative vs baseline):
- Stock avifenc still-image (Default 1T): -1.45% / -24.73% encode time
- Stock avifenc still-image (Default 8T): -0.43% / -19.16% encode time
- 48-image multi-rate corpus (Full Suite 1T): -1.65% / -25.22% time
- Bit-exact output (1T and 8T)
TAG=agy
CONV=b929a630-a6cd-45a0-979c-1767c30178ee
Change-Id: Ia2f1166e088e8141d2cc7b6a5a92d7f9e3c4acd1
diff --git a/av1/common/reconintra.c b/av1/common/reconintra.c
index 4e94c00913..a4eaf77c88 100644
--- a/av1/common/reconintra.c
+++ b/av1/common/reconintra.c
@@ -1089,42 +1089,15 @@ static void build_directional_and_filter_intra_predictors(
int i;
const uint8_t *above_ref = ref - ref_stride;
const uint8_t *left_ref = ref - 1;
- DECLARE_ALIGNED(16, uint8_t, left_data[NUM_INTRA_NEIGHBOUR_PIXELS]);
- DECLARE_ALIGNED(16, uint8_t, above_data[NUM_INTRA_NEIGHBOUR_PIXELS]);
- uint8_t *const above_row = above_data + 16;
- uint8_t *const left_col = left_data + 16;
const int txwpx = tx_size_wide[tx_size];
const int txhpx = tx_size_high[tx_size];
- int need_left = extend_modes[mode] & NEED_LEFT;
- int need_above = extend_modes[mode] & NEED_ABOVE;
- int need_above_left = extend_modes[mode] & NEED_ABOVELEFT;
const int is_dr_mode = av1_is_directional_mode(mode);
const int use_filter_intra = filter_intra_mode != FILTER_INTRA_MODES;
assert(use_filter_intra || is_dr_mode);
- // The left_data, above_data buffers must be zeroed to fix some intermittent
- // valgrind errors. Uninitialized reads in intra pred modules (e.g. width = 4
- // path in av1_dr_prediction_z1_avx2()) from left_data, above_data are seen to
- // be the potential reason for this issue.
- memset(left_data, 129, NUM_INTRA_NEIGHBOUR_PIXELS);
- memset(above_data, 127, NUM_INTRA_NEIGHBOUR_PIXELS);
-
- // The default values if ref pixels are not available:
- // 128 127 127 .. 127 127 127 127 127 127
- // 129 A B .. Y Z
- // 129 C D .. W X
- // 129 E F .. U V
- // 129 G H .. S T T T T T
- // ..
-
- if (is_dr_mode) {
- if (p_angle <= 90)
- need_above = 1, need_left = 0, need_above_left = 1;
- else if (p_angle < 180)
- need_above = 1, need_left = 1, need_above_left = 1;
- else
- need_above = 0, need_left = 1, need_above_left = 1;
- }
- if (use_filter_intra) need_left = need_above = need_above_left = 1;
+ (void)is_dr_mode;
+ const int need_above = use_filter_intra || (p_angle < 180);
+ const int need_left = use_filter_intra || (p_angle > 90);
+ const int need_above_left = 1;
assert(n_top_px >= 0);
assert(n_topright_px >= -1);
@@ -1145,22 +1118,28 @@ static void build_directional_and_filter_intra_predictors(
return;
}
+ DECLARE_ALIGNED(16, uint8_t, left_data[NUM_INTRA_NEIGHBOUR_PIXELS]);
+ DECLARE_ALIGNED(16, uint8_t, above_data[NUM_INTRA_NEIGHBOUR_PIXELS]);
+ uint8_t *const above_row = above_data + 16;
+ uint8_t *const left_col = left_data + 16;
+
// NEED_LEFT
if (need_left) {
const int num_left_pixels_needed =
txhpx + (n_bottomleft_px >= 0 ? txwpx : 0);
- i = 0;
if (n_left_px > 0) {
- for (; i < n_left_px; i++) left_col[i] = left_ref[i * ref_stride];
- if (n_bottomleft_px > 0) {
- assert(i == txhpx);
- for (; i < txhpx + n_bottomleft_px; i++)
- left_col[i] = left_ref[i * ref_stride];
- }
+ const int n_left_copy =
+ n_left_px + (n_bottomleft_px > 0 ? n_bottomleft_px : 0);
+ assert(n_left_copy > 0);
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC novector
+#endif
+ for (i = 0; i < n_left_copy; i++) left_col[i] = left_ref[i * ref_stride];
if (i < num_left_pixels_needed)
memset(&left_col[i], left_col[i - 1], num_left_pixels_needed - i);
- } else if (n_top_px > 0) {
- memset(left_col, above_ref[0], num_left_pixels_needed);
+ } else {
+ const uint8_t val = (n_top_px > 0) ? above_ref[0] : 129;
+ memset(left_col, val, num_left_pixels_needed);
}
}
@@ -1168,20 +1147,52 @@ static void build_directional_and_filter_intra_predictors(
if (need_above) {
const int num_top_pixels_needed = txwpx + (n_topright_px >= 0 ? txhpx : 0);
if (n_top_px > 0) {
- memcpy(above_row, above_ref, n_top_px);
- i = n_top_px;
- if (n_topright_px > 0) {
- assert(n_top_px == txwpx);
- memcpy(above_row + txwpx, above_ref + txwpx, n_topright_px);
- i += n_topright_px;
- }
- if (i < num_top_pixels_needed)
- memset(&above_row[i], above_row[i - 1], num_top_pixels_needed - i);
- } else if (n_left_px > 0) {
- memset(above_row, left_ref[0], num_top_pixels_needed);
+ const int n_top_copy = n_top_px + (n_topright_px > 0 ? n_topright_px : 0);
+ memcpy(above_row, above_ref, n_top_copy);
+ if (n_top_copy < num_top_pixels_needed)
+ memset(&above_row[n_top_copy], above_row[n_top_copy - 1],
+ num_top_pixels_needed - n_top_copy);
+ } else {
+ const uint8_t val = (n_left_px > 0) ? left_ref[0] : 127;
+ memset(above_row, val, num_top_pixels_needed);
}
}
+ // Pure vertical (90 deg) and horizontal (180 deg) directional predictors do
+ // not require the top-left corner pixel or intra edge filtering.
+ if (!use_filter_intra) {
+ if (p_angle == 90) {
+ pred[V_PRED][tx_size](dst, dst_stride, above_row, left_col);
+ return;
+ }
+ if (p_angle == 180) {
+ pred[H_PRED][tx_size](dst, dst_stride, above_row, left_col);
+ return;
+ }
+ }
+
+ // Zero the 16-byte prefix, any uninitialized top-right / bottom-left span
+ // (when n_topright_px < 0 or n_bottomleft_px < 0), and the SIMD over-read
+ // tail for Valgrind/MemorySanitizer.
+ const int tail_zero = (txwpx + txhpx == 128) ? 16 : 32;
+ memset(above_data, 0, 16);
+ memset(left_data, 0, 16);
+ if (need_above) {
+ const int num_top_pixels_needed = txwpx + (n_topright_px >= 0 ? txhpx : 0);
+ assert(num_top_pixels_needed >= 0 &&
+ num_top_pixels_needed <= txwpx + txhpx);
+ memset(above_row + num_top_pixels_needed, 0,
+ (txwpx + txhpx + tail_zero) - num_top_pixels_needed);
+ }
+ if (need_left) {
+ const int num_left_pixels_needed =
+ txhpx + (n_bottomleft_px >= 0 ? txwpx : 0);
+ assert(num_left_pixels_needed >= 0 &&
+ num_left_pixels_needed <= txhpx + txwpx);
+ memset(left_col + num_left_pixels_needed, 0,
+ (txhpx + txwpx + tail_zero) - num_left_pixels_needed);
+ }
+
if (need_above_left) {
if (n_top_px > 0 && n_left_px > 0) {
above_row[-1] = above_ref[-1];
@@ -1281,27 +1292,35 @@ static void build_non_directional_intra_predictors(
uint8_t *const left_col = left_data + 16;
if (need_left) {
- memset(left_data, 129, NUM_INTRA_NEIGHBOUR_PIXELS);
if (n_left_px > 0) {
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC novector
+#endif
for (i = 0; i < n_left_px; i++) left_col[i] = left_ref[i * ref_stride];
if (i < txhpx) memset(&left_col[i], left_col[i - 1], txhpx - i);
- } else if (n_top_px > 0) {
- memset(left_col, above_ref[0], txhpx);
+ } else {
+ const uint8_t val = (n_top_px > 0) ? above_ref[0] : 129;
+ memset(left_col, val, txhpx);
}
}
if (need_above) {
- memset(above_data, 127, NUM_INTRA_NEIGHBOUR_PIXELS);
if (n_top_px > 0) {
memcpy(above_row, above_ref, n_top_px);
i = n_top_px;
if (i < txwpx) memset(&above_row[i], above_row[i - 1], txwpx - i);
- } else if (n_left_px > 0) {
- memset(above_row, left_ref[0], txwpx);
+ } else {
+ const uint8_t val = (n_left_px > 0) ? left_ref[0] : 127;
+ memset(above_row, val, txwpx);
}
}
if (need_above_left) {
+ // aom_paeth_predictor_{4x4,8x4}_ssse3 load 8 bytes from left and above
+ // before shuffling with a runtime mask; zero bytes 4..7 to avoid
+ // uninitialized reads under Valgrind/MemorySanitizer (see aomedia:2752).
+ if (txhpx == 4) memset(left_col + 4, 0, 4);
+ if (txwpx == 4) memset(above_row + 4, 0, 4);
if (n_top_px > 0 && n_left_px > 0) {
above_row[-1] = above_ref[-1];
} else if (n_top_px > 0) {
@@ -1390,44 +1409,18 @@ static void highbd_build_directional_and_filter_intra_predictors(
int i;
uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
const uint16_t *const ref = CONVERT_TO_SHORTPTR(ref8);
- DECLARE_ALIGNED(16, uint16_t, left_data[NUM_INTRA_NEIGHBOUR_PIXELS]);
- DECLARE_ALIGNED(16, uint16_t, above_data[NUM_INTRA_NEIGHBOUR_PIXELS]);
- uint16_t *const above_row = above_data + 16;
- uint16_t *const left_col = left_data + 16;
const int txwpx = tx_size_wide[tx_size];
const int txhpx = tx_size_high[tx_size];
- int need_left = extend_modes[mode] & NEED_LEFT;
- int need_above = extend_modes[mode] & NEED_ABOVE;
- int need_above_left = extend_modes[mode] & NEED_ABOVELEFT;
const uint16_t *above_ref = ref - ref_stride;
const uint16_t *left_ref = ref - 1;
const int is_dr_mode = av1_is_directional_mode(mode);
const int use_filter_intra = filter_intra_mode != FILTER_INTRA_MODES;
assert(use_filter_intra || is_dr_mode);
+ (void)is_dr_mode;
const int base = 128 << (bit_depth - 8);
- // The left_data, above_data buffers must be zeroed to fix some intermittent
- // valgrind errors. Uninitialized reads in intra pred modules (e.g. width = 4
- // path in av1_highbd_dr_prediction_z2_avx2()) from left_data, above_data are
- // seen to be the potential reason for this issue.
- aom_memset16(left_data, base + 1, NUM_INTRA_NEIGHBOUR_PIXELS);
- aom_memset16(above_data, base - 1, NUM_INTRA_NEIGHBOUR_PIXELS);
-
- // The default values if ref pixels are not available:
- // base base-1 base-1 .. base-1 base-1 base-1 base-1 base-1 base-1
- // base+1 A B .. Y Z
- // base+1 C D .. W X
- // base+1 E F .. U V
- // base+1 G H .. S T T T T T
-
- if (is_dr_mode) {
- if (p_angle <= 90)
- need_above = 1, need_left = 0, need_above_left = 1;
- else if (p_angle < 180)
- need_above = 1, need_left = 1, need_above_left = 1;
- else
- need_above = 0, need_left = 1, need_above_left = 1;
- }
- if (use_filter_intra) need_left = need_above = need_above_left = 1;
+ const int need_above = use_filter_intra || (p_angle < 180);
+ const int need_left = use_filter_intra || (p_angle > 90);
+ const int need_above_left = 1;
assert(n_top_px >= 0);
assert(n_topright_px >= -1);
@@ -1448,22 +1441,28 @@ static void highbd_build_directional_and_filter_intra_predictors(
return;
}
+ DECLARE_ALIGNED(16, uint16_t, left_data[NUM_INTRA_NEIGHBOUR_PIXELS]);
+ DECLARE_ALIGNED(16, uint16_t, above_data[NUM_INTRA_NEIGHBOUR_PIXELS]);
+ uint16_t *const above_row = above_data + 16;
+ uint16_t *const left_col = left_data + 16;
+
// NEED_LEFT
if (need_left) {
const int num_left_pixels_needed =
txhpx + (n_bottomleft_px >= 0 ? txwpx : 0);
- i = 0;
if (n_left_px > 0) {
- for (; i < n_left_px; i++) left_col[i] = left_ref[i * ref_stride];
- if (n_bottomleft_px > 0) {
- assert(i == txhpx);
- for (; i < txhpx + n_bottomleft_px; i++)
- left_col[i] = left_ref[i * ref_stride];
- }
+ const int n_left_copy =
+ n_left_px + (n_bottomleft_px > 0 ? n_bottomleft_px : 0);
+ assert(n_left_copy > 0);
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC novector
+#endif
+ for (i = 0; i < n_left_copy; i++) left_col[i] = left_ref[i * ref_stride];
if (i < num_left_pixels_needed)
aom_memset16(&left_col[i], left_col[i - 1], num_left_pixels_needed - i);
- } else if (n_top_px > 0) {
- aom_memset16(left_col, above_ref[0], num_left_pixels_needed);
+ } else {
+ const int val = (n_top_px > 0) ? above_ref[0] : base + 1;
+ aom_memset16(left_col, val, num_left_pixels_needed);
}
}
@@ -1471,22 +1470,53 @@ static void highbd_build_directional_and_filter_intra_predictors(
if (need_above) {
const int num_top_pixels_needed = txwpx + (n_topright_px >= 0 ? txhpx : 0);
if (n_top_px > 0) {
- memcpy(above_row, above_ref, n_top_px * sizeof(above_ref[0]));
- i = n_top_px;
- if (n_topright_px > 0) {
- assert(n_top_px == txwpx);
- memcpy(above_row + txwpx, above_ref + txwpx,
- n_topright_px * sizeof(above_ref[0]));
- i += n_topright_px;
- }
- if (i < num_top_pixels_needed)
- aom_memset16(&above_row[i], above_row[i - 1],
- num_top_pixels_needed - i);
- } else if (n_left_px > 0) {
- aom_memset16(above_row, left_ref[0], num_top_pixels_needed);
+ const int n_top_copy = n_top_px + (n_topright_px > 0 ? n_topright_px : 0);
+ memcpy(above_row, above_ref, n_top_copy * sizeof(above_ref[0]));
+ if (n_top_copy < num_top_pixels_needed)
+ aom_memset16(&above_row[n_top_copy], above_row[n_top_copy - 1],
+ num_top_pixels_needed - n_top_copy);
+ } else {
+ const int val = (n_left_px > 0) ? left_ref[0] : base - 1;
+ aom_memset16(above_row, val, num_top_pixels_needed);
}
}
+ // Pure vertical (90 deg) and horizontal (180 deg) directional predictors do
+ // not require the top-left corner pixel or intra edge filtering.
+ if (!use_filter_intra) {
+ if (p_angle == 90) {
+ pred_high[V_PRED][tx_size](dst, dst_stride, above_row, left_col,
+ bit_depth);
+ return;
+ }
+ if (p_angle == 180) {
+ pred_high[H_PRED][tx_size](dst, dst_stride, above_row, left_col,
+ bit_depth);
+ return;
+ }
+ }
+
+ const int tail_zero = (txwpx + txhpx == 128) ? 16 : 32;
+ memset(above_data, 0, 16 * sizeof(above_data[0]));
+ memset(left_data, 0, 16 * sizeof(left_data[0]));
+ if (need_above) {
+ const int num_top_pixels_needed = txwpx + (n_topright_px >= 0 ? txhpx : 0);
+ assert(num_top_pixels_needed >= 0 &&
+ num_top_pixels_needed <= txwpx + txhpx);
+ memset(above_row + num_top_pixels_needed, 0,
+ ((txwpx + txhpx + tail_zero) - num_top_pixels_needed) *
+ sizeof(above_row[0]));
+ }
+ if (need_left) {
+ const int num_left_pixels_needed =
+ txhpx + (n_bottomleft_px >= 0 ? txwpx : 0);
+ assert(num_left_pixels_needed >= 0 &&
+ num_left_pixels_needed <= txhpx + txwpx);
+ memset(left_col + num_left_pixels_needed, 0,
+ ((txhpx + txwpx + tail_zero) - num_left_pixels_needed) *
+ sizeof(left_col[0]));
+ }
+
if (need_above_left) {
if (n_top_px > 0 && n_left_px > 0) {
above_row[-1] = above_ref[-1];
@@ -1592,23 +1622,26 @@ static void highbd_build_non_directional_intra_predictors(
uint16_t *const left_col = left_data + 16;
if (need_left) {
- aom_memset16(left_data, base + 1, NUM_INTRA_NEIGHBOUR_PIXELS);
if (n_left_px > 0) {
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC novector
+#endif
for (i = 0; i < n_left_px; i++) left_col[i] = left_ref[i * ref_stride];
if (i < txhpx) aom_memset16(&left_col[i], left_col[i - 1], txhpx - i);
- } else if (n_top_px > 0) {
- aom_memset16(left_col, above_ref[0], txhpx);
+ } else {
+ const int val = (n_top_px > 0) ? above_ref[0] : base + 1;
+ aom_memset16(left_col, val, txhpx);
}
}
if (need_above) {
- aom_memset16(above_data, base - 1, NUM_INTRA_NEIGHBOUR_PIXELS);
if (n_top_px > 0) {
memcpy(above_row, above_ref, n_top_px * sizeof(above_ref[0]));
i = n_top_px;
if (i < txwpx) aom_memset16(&above_row[i], above_row[i - 1], (txwpx - i));
- } else if (n_left_px > 0) {
- aom_memset16(above_row, left_ref[0], txwpx);
+ } else {
+ const int val = (n_left_px > 0) ? left_ref[0] : base - 1;
+ aom_memset16(above_row, val, txwpx);
}
}
@@ -1685,14 +1718,40 @@ static inline BLOCK_SIZE scale_chroma_bsize(BLOCK_SIZE bsize, int subsampling_x,
return bs;
}
-void av1_predict_intra_block(const MACROBLOCKD *xd, BLOCK_SIZE sb_size,
- int enable_intra_edge_filter, int wpx, int hpx,
- TX_SIZE tx_size, PREDICTION_MODE mode,
- int angle_delta, int use_palette,
- FILTER_INTRA_MODE filter_intra_mode,
- const uint8_t *ref, int ref_stride, uint8_t *dst,
- int dst_stride, int col_off, int row_off,
- int plane) {
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
+static void predict_palette_block(const MACROBLOCKD *xd, int wpx, int txwpx,
+ int txhpx, int x, int y, uint8_t *dst,
+ int dst_stride, int plane, int is_hbd) {
+ int r, c;
+ const MB_MODE_INFO *const mbmi = xd->mi[0];
+ const uint8_t *const map = xd->plane[plane != 0].color_index_map +
+ xd->color_index_map_offset[plane != 0];
+ const uint16_t *const palette =
+ mbmi->palette_mode_info.palette_colors + plane * PALETTE_MAX_SIZE;
+ if (is_hbd) {
+ uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst);
+ for (r = 0; r < txhpx; ++r) {
+ for (c = 0; c < txwpx; ++c) {
+ dst16[r * dst_stride + c] = palette[map[(r + y) * wpx + c + x]];
+ }
+ }
+ } else {
+ for (r = 0; r < txhpx; ++r) {
+ for (c = 0; c < txwpx; ++c) {
+ dst[r * dst_stride + c] = (uint8_t)palette[map[(r + y) * wpx + c + x]];
+ }
+ }
+ }
+}
+
+static AOM_FORCE_INLINE void predict_intra_block_internal(
+ const MACROBLOCKD *xd, BLOCK_SIZE sb_size, int enable_intra_edge_filter,
+ int wpx, int hpx, TX_SIZE tx_size, PREDICTION_MODE mode, int angle_delta,
+ int use_palette, FILTER_INTRA_MODE filter_intra_mode, const uint8_t *ref,
+ int ref_stride, uint8_t *dst, int dst_stride, int col_off, int row_off,
+ int plane) {
const MB_MODE_INFO *const mbmi = xd->mi[0];
const int txwpx = tx_size_wide[tx_size];
const int txhpx = tx_size_high[tx_size];
@@ -1702,27 +1761,9 @@ void av1_predict_intra_block(const MACROBLOCKD *xd, BLOCK_SIZE sb_size,
assert(mode < INTRA_MODES);
- if (use_palette) {
- int r, c;
- const uint8_t *const map = xd->plane[plane != 0].color_index_map +
- xd->color_index_map_offset[plane != 0];
- const uint16_t *const palette =
- mbmi->palette_mode_info.palette_colors + plane * PALETTE_MAX_SIZE;
- if (is_hbd) {
- uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst);
- for (r = 0; r < txhpx; ++r) {
- for (c = 0; c < txwpx; ++c) {
- dst16[r * dst_stride + c] = palette[map[(r + y) * wpx + c + x]];
- }
- }
- } else {
- for (r = 0; r < txhpx; ++r) {
- for (c = 0; c < txwpx; ++c) {
- dst[r * dst_stride + c] =
- (uint8_t)palette[map[(r + y) * wpx + c + x]];
- }
- }
- }
+ if (UNLIKELY(use_palette)) {
+ predict_palette_block(xd, wpx, txwpx, txhpx, x, y, dst, dst_stride, plane,
+ is_hbd);
return;
}
@@ -1743,13 +1784,6 @@ void av1_predict_intra_block(const MACROBLOCKD *xd, BLOCK_SIZE sb_size,
const int use_filter_intra = filter_intra_mode != FILTER_INTRA_MODES;
const int is_dr_mode = av1_is_directional_mode(mode);
- // The computations in this function, as well as in build_intra_predictors(),
- // are generalized for all intra modes. Some of these operations are not
- // required since non-directional intra modes (i.e., DC, SMOOTH, SMOOTH_H,
- // SMOOTH_V, and PAETH) specifically require left and top neighbors. Hence, a
- // separate function build_non_directional_intra_predictors() is introduced
- // for these modes to avoid redundant computations while generating pred data.
-
const int n_top_px = have_top ? clamp(xr + txwpx, 0, txwpx) : 0;
const int n_left_px = have_left ? clamp(yd + txhpx, 0, txhpx) : 0;
if (!use_filter_intra && !is_dr_mode) {
@@ -1766,58 +1800,53 @@ void av1_predict_intra_block(const MACROBLOCKD *xd, BLOCK_SIZE sb_size,
return;
}
- const int txw = tx_size_wide_unit[tx_size];
- const int txh = tx_size_high_unit[tx_size];
- const int mi_row = -xd->mb_to_top_edge >> (3 + MI_SIZE_LOG2);
- const int mi_col = -xd->mb_to_left_edge >> (3 + MI_SIZE_LOG2);
- const int right_available =
- mi_col + ((col_off + txw) << ss_x) < xd->tile.mi_col_end;
- const int bottom_available =
- (yd > 0) && (mi_row + ((row_off + txh) << ss_y) < xd->tile.mi_row_end);
-
- const PARTITION_TYPE partition = mbmi->partition;
-
- BLOCK_SIZE bsize = mbmi->bsize;
- // force 4x4 chroma component block size.
- if (ss_x || ss_y) {
- bsize = scale_chroma_bsize(bsize, ss_x, ss_y);
- }
-
int p_angle = 0;
- int need_top_right = extend_modes[mode] & NEED_ABOVERIGHT;
- int need_bottom_left = extend_modes[mode] & NEED_BOTTOMLEFT;
+ int n_topright_px = -1;
+ int n_bottomleft_px = -1;
- if (use_filter_intra) {
- need_top_right = 0;
- need_bottom_left = 0;
- }
if (is_dr_mode) {
p_angle = mode_to_angle_map[mode] + angle_delta;
- need_top_right = p_angle < 90;
- need_bottom_left = p_angle > 180;
+ if (p_angle < 90) {
+ n_topright_px = 0;
+ if (have_top && xr > 0) {
+ const int mi_col = -xd->mb_to_left_edge >> (3 + MI_SIZE_LOG2);
+ const int txw = tx_size_wide_unit[tx_size];
+ if (mi_col + ((col_off + txw) << ss_x) < xd->tile.mi_col_end) {
+ const int mi_row = -xd->mb_to_top_edge >> (3 + MI_SIZE_LOG2);
+ BLOCK_SIZE bsize = mbmi->bsize;
+ if (ss_x || ss_y) bsize = scale_chroma_bsize(bsize, ss_x, ss_y);
+ if (has_top_right(sb_size, bsize, mi_row, mi_col, 1, 1,
+ mbmi->partition, tx_size, row_off, col_off, ss_x,
+ ss_y)) {
+ n_topright_px = AOMMIN(xr, txwpx);
+ }
+ }
+ }
+ } else if (p_angle > 180) {
+ n_bottomleft_px = 0;
+ if (have_left && yd > 0) {
+ const int mi_row = -xd->mb_to_top_edge >> (3 + MI_SIZE_LOG2);
+ const int txh = tx_size_high_unit[tx_size];
+ if (mi_row + ((row_off + txh) << ss_y) < xd->tile.mi_row_end) {
+ const int mi_col = -xd->mb_to_left_edge >> (3 + MI_SIZE_LOG2);
+ BLOCK_SIZE bsize = mbmi->bsize;
+ if (ss_x || ss_y) bsize = scale_chroma_bsize(bsize, ss_x, ss_y);
+ if (has_bottom_left(sb_size, bsize, mi_row, mi_col, 1, 1,
+ mbmi->partition, tx_size, row_off, col_off, ss_x,
+ ss_y)) {
+ n_bottomleft_px = AOMMIN(yd, txhpx);
+ }
+ }
+ }
+ }
}
- // Possible states for have_top_right(TR) and have_bottom_left(BL)
- // -1 : TR and BL are not needed
- // 0 : TR and BL are needed but not available
- // > 0 : TR and BL are needed and pixels are available
- const int have_top_right =
- need_top_right ? has_top_right(sb_size, bsize, mi_row, mi_col, have_top,
- right_available, partition, tx_size,
- row_off, col_off, ss_x, ss_y)
- : -1;
- const int have_bottom_left =
- need_bottom_left ? has_bottom_left(sb_size, bsize, mi_row, mi_col,
- bottom_available, have_left, partition,
- tx_size, row_off, col_off, ss_x, ss_y)
- : -1;
-
const int disable_edge_filter = !enable_intra_edge_filter;
- const int intra_edge_filter_type = get_intra_edge_filter_type(xd, plane);
- const int n_topright_px =
- have_top_right > 0 ? clamp(xr, 0, txwpx) : have_top_right;
- const int n_bottomleft_px =
- have_bottom_left > 0 ? clamp(yd, 0, txhpx) : have_bottom_left;
+ const int intra_edge_filter_type =
+ (!disable_edge_filter && !use_filter_intra && p_angle != 90 &&
+ p_angle != 180)
+ ? get_intra_edge_filter_type(xd, plane)
+ : 0;
#if CONFIG_AV1_HIGHBITDEPTH
if (is_hbd) {
highbd_build_directional_and_filter_intra_predictors(
@@ -1833,6 +1862,20 @@ void av1_predict_intra_block(const MACROBLOCKD *xd, BLOCK_SIZE sb_size,
n_bottomleft_px, intra_edge_filter_type);
}
+void av1_predict_intra_block(const MACROBLOCKD *xd, BLOCK_SIZE sb_size,
+ int enable_intra_edge_filter, int wpx, int hpx,
+ TX_SIZE tx_size, PREDICTION_MODE mode,
+ int angle_delta, int use_palette,
+ FILTER_INTRA_MODE filter_intra_mode,
+ const uint8_t *ref, int ref_stride, uint8_t *dst,
+ int dst_stride, int col_off, int row_off,
+ int plane) {
+ predict_intra_block_internal(xd, sb_size, enable_intra_edge_filter, wpx, hpx,
+ tx_size, mode, angle_delta, use_palette,
+ filter_intra_mode, ref, ref_stride, dst,
+ dst_stride, col_off, row_off, plane);
+}
+
void av1_predict_intra_block_facade(const AV1_COMMON *cm, MACROBLOCKD *xd,
int plane, int blk_col, int blk_row,
TX_SIZE tx_size) {
@@ -1851,7 +1894,7 @@ void av1_predict_intra_block_facade(const AV1_COMMON *cm, MACROBLOCKD *xd,
const SequenceHeader *seq_params = cm->seq_params;
#if !CONFIG_REALTIME_ONLY || CONFIG_AV1_DECODER
- if (plane != AOM_PLANE_Y && mbmi->uv_mode == UV_CFL_PRED) {
+ if (UNLIKELY(plane != AOM_PLANE_Y && mbmi->uv_mode == UV_CFL_PRED)) {
#if CONFIG_DEBUG
assert(is_cfl_allowed(xd));
const BLOCK_SIZE plane_bsize =
@@ -1868,11 +1911,11 @@ void av1_predict_intra_block_facade(const AV1_COMMON *cm, MACROBLOCKD *xd,
CFL_CTX *const cfl = &xd->cfl;
CFL_PRED_TYPE pred_plane = get_cfl_pred_type(plane);
if (!cfl->dc_pred_is_cached[pred_plane]) {
- av1_predict_intra_block(xd, seq_params->sb_size,
- seq_params->enable_intra_edge_filter, pd->width,
- pd->height, tx_size, mode, angle_delta,
- use_palette, filter_intra_mode, dst, dst_stride,
- dst, dst_stride, blk_col, blk_row, plane);
+ predict_intra_block_internal(
+ xd, seq_params->sb_size, seq_params->enable_intra_edge_filter,
+ pd->width, pd->height, tx_size, mode, angle_delta, use_palette,
+ filter_intra_mode, dst, dst_stride, dst, dst_stride, blk_col, blk_row,
+ plane);
if (cfl->use_dc_pred_cache) {
cfl_store_dc_pred(xd, dst, pred_plane, tx_size_wide[tx_size]);
cfl->dc_pred_is_cached[pred_plane] = true;
@@ -1884,7 +1927,7 @@ void av1_predict_intra_block_facade(const AV1_COMMON *cm, MACROBLOCKD *xd,
return;
}
#endif // !CONFIG_REALTIME_ONLY || CONFIG_AV1_DECODER
- av1_predict_intra_block(
+ predict_intra_block_internal(
xd, seq_params->sb_size, seq_params->enable_intra_edge_filter, pd->width,
pd->height, tx_size, mode, angle_delta, use_palette, filter_intra_mode,
dst, dst_stride, dst, dst_stride, blk_col, blk_row, plane);
diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c
index 42d528b4f5..5d8da98dd3 100644
--- a/av1/encoder/tx_search.c
+++ b/av1/encoder/tx_search.c
@@ -1150,6 +1150,9 @@ static inline void dist_block_tx_domain(MACROBLOCK *x, int plane, int block,
*out_sse = RIGHT_SIGNED_SHIFT(this_sse, shift);
}
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
static uint16_t prune_txk_type_separ(
const AV1_COMP *cpi, MACROBLOCK *x, int plane, int block, TX_SIZE tx_size,
int blk_row, int blk_col, BLOCK_SIZE plane_bsize, int *txk_map,
@@ -1166,19 +1169,17 @@ static uint16_t prune_txk_type_separ(
int idx_h[4] = { 0, 1, 2, 3 };
int skip_v[4] = { 0 };
int skip_h[4] = { 0 };
- const int idx_map[16] = {
+ static const int idx_map[16] = {
DCT_DCT, DCT_ADST, DCT_FLIPADST, V_DCT,
ADST_DCT, ADST_ADST, ADST_FLIPADST, V_ADST,
FLIPADST_DCT, FLIPADST_ADST, FLIPADST_FLIPADST, V_FLIPADST,
H_DCT, H_ADST, H_FLIPADST, IDTX
};
- const int sel_pattern_v[16] = {
- 0, 0, 1, 1, 0, 2, 1, 2, 2, 0, 3, 1, 3, 2, 3, 3
- };
- const int sel_pattern_h[16] = {
- 0, 1, 0, 1, 2, 0, 2, 1, 2, 3, 0, 3, 1, 3, 2, 3
- };
+ static const int sel_pattern_v[16] = { 0, 0, 1, 1, 0, 2, 1, 2,
+ 2, 0, 3, 1, 3, 2, 3, 3 };
+ static const int sel_pattern_h[16] = { 0, 1, 0, 1, 2, 0, 2, 1,
+ 2, 3, 0, 3, 1, 3, 2, 3 };
QUANT_PARAM quant_param;
TxfmParam txfm_param;
@@ -1294,6 +1295,9 @@ static uint16_t prune_txk_type_separ(
return prune;
}
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
static uint16_t prune_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
int block, TX_SIZE tx_size, int blk_row,
int blk_col, BLOCK_SIZE plane_bsize,
@@ -1428,7 +1432,7 @@ static const float *prune_2D_adaptive_thresholds[] = {
static inline float get_adaptive_thresholds(
TX_SIZE tx_size, TxSetType tx_set_type,
TX_TYPE_PRUNE_MODE prune_2d_txfm_mode) {
- const int prune_aggr_table[5][2] = {
+ static const int prune_aggr_table[5][2] = {
{ 4, 1 }, { 6, 3 }, { 9, 6 }, { 9, 6 }, { 12, 9 }
};
int pruning_aggressiveness = 0;
@@ -1503,10 +1507,13 @@ static inline void unset_bit_mask(uint16_t *mask, int val) {
*mask &= ~(1 << val);
}
+#if defined(__GNUC__) || defined(__clang__)
+__attribute__((noinline))
+#endif
static void prune_tx_2D(MACROBLOCK *x, BLOCK_SIZE bsize, TX_SIZE tx_size,
int blk_row, int blk_col, TxSetType tx_set_type,
TX_TYPE_PRUNE_MODE prune_2d_txfm_mode, int *txk_map,
- uint16_t *allowed_tx_mask) {
+ uint16_t *allowed_tx_mask, int *is_txk_map_populated) {
// This table is used because the search order is different from the enum
// order.
static const int tx_type_table_2D[16] = {
@@ -1618,6 +1625,7 @@ static void prune_tx_2D(MACROBLOCK *x, BLOCK_SIZE bsize, TX_SIZE tx_size,
// end the search.
set_bit_mask(&allow_bitmask, tx_type_table_2D[max_score_i]);
memcpy(txk_map, tx_type_table_2D, sizeof(tx_type_table_2D));
+ *is_txk_map_populated = 1;
*allowed_tx_mask = allow_bitmask;
return;
}
@@ -1656,6 +1664,7 @@ static void prune_tx_2D(MACROBLOCK *x, BLOCK_SIZE bsize, TX_SIZE tx_size,
}
memcpy(txk_map, tx_type_allowed, sizeof(tx_type_table_2D));
+ *is_txk_map_populated = 1;
*allowed_tx_mask = allow_bitmask;
}
@@ -1742,7 +1751,8 @@ static inline uint16_t get_tx_mask(
const AV1_COMP *cpi, MACROBLOCK *x, int plane, int block, int blk_row,
int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
const TXB_CTX *const txb_ctx, FAST_TX_SEARCH_MODE ftxs_mode,
- int64_t ref_best_rd, TX_TYPE *allowed_txk_types, int *txk_map) {
+ int64_t ref_best_rd, TX_TYPE *allowed_txk_types, int *txk_map,
+ int *is_txk_map_populated) {
const AV1_COMMON *cm = &cpi->common;
MACROBLOCKD *xd = &x->e_mbd;
MB_MODE_INFO *mbmi = xd->mi[0];
@@ -1752,6 +1762,7 @@ static inline uint16_t get_tx_mask(
// if txk_allowed = TX_TYPES, >1 tx types are allowed, else, if txk_allowed <
// TX_TYPES, only that specific tx type is allowed.
TX_TYPE txk_allowed = TX_TYPES;
+ *is_txk_map_populated = 0;
const FRAME_UPDATE_TYPE update_type =
get_frame_update_type(&cpi->ppi->gf_group, cpi->gf_frame_index);
@@ -1868,42 +1879,47 @@ static inline uint16_t get_tx_mask(
if ((prune >> max_idx) & 0x01) prune &= ~(1 << max_idx);
allowed_tx_mask &= (~prune);
}
+
+ if (cpi->sf.tx_sf.tx_type_search.prune_tx_type_est_rd ||
+ (txfm_params->prune_2d_txfm_mode >= TX_TYPE_PRUNE_1 && is_inter)) {
#if defined(__GNUC__) || defined(__clang__)
- num_allowed = __builtin_popcount(allowed_tx_mask);
+ num_allowed = __builtin_popcount(allowed_tx_mask);
#else
- for (uint16_t mask = allowed_tx_mask; mask != 0; mask &= mask - 1) {
- num_allowed++;
- }
-#endif
- assert(num_allowed > 0);
-
- if (num_allowed > 2 && cpi->sf.tx_sf.tx_type_search.prune_tx_type_est_rd) {
- int pf = prune_factors[txfm_params->prune_2d_txfm_mode];
- int mf = mul_factors[txfm_params->prune_2d_txfm_mode];
- if (num_allowed <= 7) {
- const uint16_t prune =
- prune_txk_type(cpi, x, plane, block, tx_size, blk_row, blk_col,
- plane_bsize, txk_map, allowed_tx_mask, pf, txb_ctx,
- cm->features.reduced_tx_set_used);
- allowed_tx_mask &= (~prune);
- } else {
- const int num_sel = (num_allowed * mf + 50) / 100;
- const uint16_t prune = prune_txk_type_separ(
- cpi, x, plane, block, tx_size, blk_row, blk_col, plane_bsize,
- txk_map, allowed_tx_mask, pf, txb_ctx,
- cm->features.reduced_tx_set_used, ref_best_rd, num_sel);
-
- allowed_tx_mask &= (~prune);
+ for (uint16_t mask = allowed_tx_mask; mask != 0; mask &= mask - 1) {
+ num_allowed++;
}
- } else {
+#endif
assert(num_allowed > 0);
- int allowed_tx_count =
- (txfm_params->prune_2d_txfm_mode >= TX_TYPE_PRUNE_4) ? 1 : 5;
- // !fast_tx_search && txk_end != txk_start && plane == 0
- if (txfm_params->prune_2d_txfm_mode >= TX_TYPE_PRUNE_1 && is_inter &&
- num_allowed > allowed_tx_count) {
- prune_tx_2D(x, plane_bsize, tx_size, blk_row, blk_col, tx_set_type,
- txfm_params->prune_2d_txfm_mode, txk_map, &allowed_tx_mask);
+ if (num_allowed > 2 &&
+ cpi->sf.tx_sf.tx_type_search.prune_tx_type_est_rd) {
+ int pf = prune_factors[txfm_params->prune_2d_txfm_mode];
+ int mf = mul_factors[txfm_params->prune_2d_txfm_mode];
+ if (num_allowed <= 7) {
+ const uint16_t prune =
+ prune_txk_type(cpi, x, plane, block, tx_size, blk_row, blk_col,
+ plane_bsize, txk_map, allowed_tx_mask, pf, txb_ctx,
+ cm->features.reduced_tx_set_used);
+ allowed_tx_mask &= (~prune);
+ } else {
+ const int num_sel = (num_allowed * mf + 50) / 100;
+ const uint16_t prune = prune_txk_type_separ(
+ cpi, x, plane, block, tx_size, blk_row, blk_col, plane_bsize,
+ txk_map, allowed_tx_mask, pf, txb_ctx,
+ cm->features.reduced_tx_set_used, ref_best_rd, num_sel);
+
+ allowed_tx_mask &= (~prune);
+ }
+ *is_txk_map_populated = 1;
+ } else if (txfm_params->prune_2d_txfm_mode >= TX_TYPE_PRUNE_1 &&
+ is_inter) {
+ int allowed_tx_count =
+ (txfm_params->prune_2d_txfm_mode >= TX_TYPE_PRUNE_4) ? 1 : 5;
+ // !fast_tx_search && txk_end != txk_start && plane == 0
+ if (num_allowed > allowed_tx_count) {
+ prune_tx_2D(x, plane_bsize, tx_size, blk_row, blk_col, tx_set_type,
+ txfm_params->prune_2d_txfm_mode, txk_map,
+ &allowed_tx_mask, is_txk_map_populated);
+ }
}
}
}
@@ -2077,9 +2093,8 @@ static void search_tx_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
// txk_allowed = TX_TYPES: >1 tx types are allowed
// txk_allowed < TX_TYPES: only that specific tx type is allowed.
TX_TYPE txk_allowed = TX_TYPES;
- int txk_map[TX_TYPES] = {
- 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
- };
+ int txk_map[TX_TYPES];
+ int is_txk_map_populated = 0;
const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
const int qstep = x->plane[plane].dequant_QTX[1] >> dequant_shift;
@@ -2126,7 +2141,7 @@ static void search_tx_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
else
tx_mask = get_tx_mask(cpi, x, plane, block, blk_row, blk_col, plane_bsize,
tx_size, txb_ctx, ftxs_mode, ref_best_rd,
- &txk_allowed, txk_map);
+ &txk_allowed, txk_map, &is_txk_map_populated);
const uint16_t allowed_tx_mask = tx_mask;
if (is_cur_buf_hbd(xd)) {
@@ -2202,6 +2217,20 @@ static void search_tx_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
if (allowed_tx_mask != 0 && (allowed_tx_mask & (allowed_tx_mask - 1)) == 0) {
txk_map[0] = get_msb(allowed_tx_mask);
num_tx_candidates = 1;
+ } else if (allowed_tx_mask != 0 && !is_txk_map_populated) {
+ // Populate txk_map directly from the set bits of allowed_tx_mask when no
+ // pruning step has reordered the candidate list.
+ uint32_t mask = allowed_tx_mask;
+ num_tx_candidates = 0;
+ do {
+ txk_map[num_tx_candidates++] =
+#if defined(__GNUC__) || defined(__clang__)
+ __builtin_ctz(mask);
+#else
+ aom_clzll(mask & -(int64_t)mask) ^ 63;
+#endif
+ mask &= mask - 1;
+ } while (mask);
}
uint16_t rem_tx_mask = allowed_tx_mask;
// Iterate through all transform type candidates.
@@ -3122,13 +3151,7 @@ static inline void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
// given plane/channel, and calculate the corresponding RD cost.
static inline void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
- void *arg) {
- struct rdcost_block_args *args = arg;
- if (args->exit_early) {
- args->incomplete_exit = 1;
- return;
- }
-
+ struct rdcost_block_args *args) {
MACROBLOCK *const x = args->x;
MACROBLOCKD *const xd = &x->e_mbd;
const int is_inter = is_inter_block(xd->mi[0]);
@@ -3225,12 +3248,15 @@ int64_t av1_estimate_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
const int max_blocks_high = max_block_high(xd, bs, 0);
struct rdcost_block_args args;
- av1_zero(args);
args.x = x;
args.cpi = cpi;
args.best_rd = ref_best_rd;
args.current_rd = AOMMIN(no_this_rd, skip_txfm_rd);
+ args.exit_early = 0;
+ args.incomplete_exit = 0;
av1_init_rd_stats(&args.rd_stats);
+ av1_zero(args.t_above);
+ av1_zero(args.t_left);
av1_get_entropy_contexts(bs, &xd->plane[0], args.t_above, args.t_left);
int i = 0;
for (int blk_row = 0; blk_row < max_blocks_high && !args.incomplete_exit;
@@ -3829,18 +3855,60 @@ void av1_txfm_rd_in_plane(MACROBLOCK *x, const AV1_COMP *cpi,
MACROBLOCKD *const xd = &x->e_mbd;
const struct macroblockd_plane *const pd = &xd->plane[plane];
struct rdcost_block_args args;
- av1_zero(args);
args.x = x;
args.cpi = cpi;
args.best_rd = ref_best_rd;
args.current_rd = current_rd;
+ args.exit_early = 0;
+ args.incomplete_exit = 0;
args.ftxs_mode = ftxs_mode;
args.skip_trellis = 0;
av1_init_rd_stats(&args.rd_stats);
+ av1_zero(args.t_above);
+ av1_zero(args.t_left);
av1_get_entropy_contexts(plane_bsize, pd, args.t_above, args.t_left);
- av1_foreach_transformed_block_in_plane(xd, plane_bsize, plane, block_rd_txfm,
- &args);
+ // Fast path when the transform size matches the plane block size (single
+ // transform block); otherwise iterate over all transform blocks in plane.
+ const TX_SIZE plane_tx_size = av1_get_tx_size(plane, xd);
+ const BLOCK_SIZE tx_bsize = txsize_to_bsize[plane_tx_size];
+ if (plane_bsize == tx_bsize) {
+ block_rd_txfm(plane, 0, 0, 0, plane_bsize, plane_tx_size, &args);
+ } else {
+ const uint8_t txw_unit = tx_size_wide_unit[plane_tx_size];
+ const uint8_t txh_unit = tx_size_high_unit[plane_tx_size];
+ const int step = txw_unit * txh_unit;
+ const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
+ const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
+ const BLOCK_SIZE max_unit_bsize =
+ get_plane_block_size(BLOCK_64X64, pd->subsampling_x, pd->subsampling_y);
+ const int mu_blocks_wide =
+ AOMMIN(mi_size_wide[max_unit_bsize], max_blocks_wide);
+ const int mu_blocks_high =
+ AOMMIN(mi_size_high[max_unit_bsize], max_blocks_high);
+
+ int i = 0;
+ for (int r = 0; r < max_blocks_high && !args.incomplete_exit;
+ r += mu_blocks_high) {
+ const int unit_height = AOMMIN(mu_blocks_high + r, max_blocks_high);
+ for (int c = 0; c < max_blocks_wide && !args.incomplete_exit;
+ c += mu_blocks_wide) {
+ const int unit_width = AOMMIN(mu_blocks_wide + c, max_blocks_wide);
+ for (int blk_row = r; blk_row < unit_height && !args.incomplete_exit;
+ blk_row += txh_unit) {
+ for (int blk_col = c; blk_col < unit_width; blk_col += txw_unit) {
+ if (args.exit_early) {
+ args.incomplete_exit = 1;
+ break;
+ }
+ block_rd_txfm(plane, i, blk_row, blk_col, plane_bsize,
+ plane_tx_size, &args);
+ i += step;
+ }
+ }
+ }
+ }
+ }
MB_MODE_INFO *const mbmi = xd->mi[0];
const int is_inter = is_inter_block(mbmi);