Commit 92094f8c5f for aom
commit 92094f8c5f72ed193541d87409761b5c53822d48
Author: Jerome Jiang <jianj@google.com>
Date: Fri Jul 24 13:26:35 2026 -0400
Convolve 2d: fix mismatch in hwy
This showed up in nightly avx512 test.
AV1/AVxEncoderThreadTest.EncoderResultTest reports mismatch.
In `ConvolveVerticalPass`, 2 output rows are processed per iteration.
When `h < 8`, the loop iteration limit was previously hardcoded
to 2 when `h_ge_8` was false.
For $h = 2$ blocks, executing 2 iterations processed 4 rows, writing 2
out-of-bounds rows past the destination buffer boundary.
Also add asan poisoning for the guard region in the unit test.
This helps catching overwriting extra rows beyond height
Change-Id: I19d50c62b2f3f2b3578db43cba106e12c66e3a3e
diff --git a/av1/common/convolve_2d_sr_hwy.h b/av1/common/convolve_2d_sr_hwy.h
index 6547f26f4c..91bfa0789b 100644
--- a/av1/common/convolve_2d_sr_hwy.h
+++ b/av1/common/convolve_2d_sr_hwy.h
@@ -323,18 +323,20 @@ HWY_ATTR HWY_INLINE hn::VFromD<D> LoadAndCombineClamped(D d, D128 d128,
// from `strip_im_buf` and instead compute horizontal convolve on the fly.
// - `round_0_const`, `round_1_const`: Compile-time rounding offsets, enabling
// optimized shift calculations when non-zero.
+// - `limit_val`: Compile-time iteration limit (1 for h=2, 2 for h=4, 3 for h=6,
+// 4 for h>=8).
// - `IdxTbl`: Target-specific Highway type for table lookup indices.
// - `ComputeHRowBlock`: Callable type for computing horizontal convolve on the
// fly (used when `skip_strip_im` is true).
template <int taps_y_val, bool h_ge_8, bool skip_strip_im, int round_0_const,
- int round_1_const, class IdxTbl,
+ int round_1_const, int limit_val, class IdxTbl,
class ComputeHRowBlock = IVec16 (*)(int)>
HWY_ATTR HWY_INLINE void ConvolveVerticalPass(
const int16_t *strip_im_buf, uint8_t *dst, int dst_stride, int w, int h,
const IVec16 *coeffs_v, IVec32 round_const_y, int round_1, int bits,
IdxTbl idx_tbl, ComputeHRowBlock compute_h_row_block = nullptr) {
constexpr int num_coeffs = taps_y_val / 2;
- constexpr int num_z = (h_ge_8 ? 8 : 4) / 2 + num_coeffs - 1;
+ constexpr int num_z = limit_val + num_coeffs - 1;
constexpr int num_h = num_z / 2 + 1;
hn::Half<decltype(int16xN_tag)> d16_16;
@@ -396,8 +398,7 @@ HWY_ATTR HWY_INLINE void ConvolveVerticalPass(
Z[z_idx] = hn::TableLookupLanes(L, idx_tbl);
}
- constexpr int limit = h_ge_8 ? 4 : 2;
- for (int i = 0; i < limit; ++i) {
+ for (int i = 0; i < limit_val; ++i) {
auto sum = ConvolveVertical<taps_y_val>(int32xN_tag, &Z[i], coeffs_v,
round_const_y);
round_and_store(sum, y + 2 * i);
@@ -580,25 +581,29 @@ HWY_ATTR HWY_INLINE void Convolve2DSRHwyImpl(
HWY_IF_CONSTEXPR(taps_y_const > 0) {
constexpr bool skip_strip_im = taps_y_const == 12 ? false : true;
ConvolveVerticalPass<taps_y_const, false, skip_strip_im, round_0_const,
- round_1_const>(
+ round_1_const, /*limit_val=*/2>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
round_1, bits, idx_tbl, compute_h_row_block);
}
else {
if (taps_y == 2) {
- ConvolveVerticalPass<2, false, true, round_0_const, round_1_const>(
+ ConvolveVerticalPass<2, false, true, round_0_const, round_1_const,
+ /*limit_val=*/2>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
round_1, bits, idx_tbl, compute_h_row_block);
} else if (taps_y == 4) {
- ConvolveVerticalPass<4, false, true, round_0_const, round_1_const>(
+ ConvolveVerticalPass<4, false, true, round_0_const, round_1_const,
+ /*limit_val=*/2>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
round_1, bits, idx_tbl, compute_h_row_block);
} else if (taps_y == 6) {
- ConvolveVerticalPass<6, false, true, round_0_const, round_1_const>(
+ ConvolveVerticalPass<6, false, true, round_0_const, round_1_const,
+ /*limit_val=*/2>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
round_1, bits, idx_tbl, compute_h_row_block);
} else if (taps_y == 8) {
- ConvolveVerticalPass<8, false, true, round_0_const, round_1_const>(
+ ConvolveVerticalPass<8, false, true, round_0_const, round_1_const,
+ /*limit_val=*/2>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
round_1, bits, idx_tbl, compute_h_row_block);
}
@@ -607,25 +612,29 @@ HWY_ATTR HWY_INLINE void Convolve2DSRHwyImpl(
HWY_IF_CONSTEXPR(taps_y_const > 0) {
constexpr bool skip_strip_im = taps_y_const == 12 ? false : true;
ConvolveVerticalPass<taps_y_const, true, skip_strip_im, round_0_const,
- round_1_const>(
+ round_1_const, /*limit_val=*/4>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
round_1, bits, idx_tbl, compute_h_row_block);
}
else {
if (taps_y == 2) {
- ConvolveVerticalPass<2, true, true, round_0_const, round_1_const>(
+ ConvolveVerticalPass<2, true, true, round_0_const, round_1_const,
+ /*limit_val=*/4>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
round_1, bits, idx_tbl, compute_h_row_block);
} else if (taps_y == 4) {
- ConvolveVerticalPass<4, true, true, round_0_const, round_1_const>(
+ ConvolveVerticalPass<4, true, true, round_0_const, round_1_const,
+ /*limit_val=*/4>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
round_1, bits, idx_tbl, compute_h_row_block);
} else if (taps_y == 6) {
- ConvolveVerticalPass<6, true, true, round_0_const, round_1_const>(
+ ConvolveVerticalPass<6, true, true, round_0_const, round_1_const,
+ /*limit_val=*/4>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
round_1, bits, idx_tbl, compute_h_row_block);
} else if (taps_y == 8) {
- ConvolveVerticalPass<8, true, true, round_0_const, round_1_const>(
+ ConvolveVerticalPass<8, true, true, round_0_const, round_1_const,
+ /*limit_val=*/4>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
round_1, bits, idx_tbl, compute_h_row_block);
}
@@ -633,74 +642,152 @@ HWY_ATTR HWY_INLINE void Convolve2DSRHwyImpl(
} else {
if (h >= 8) {
if (is_taps_y_12) {
- ConvolveVerticalPass<12, true, false, round_0_const, round_1_const>(
+ ConvolveVerticalPass<12, true, false, round_0_const, round_1_const,
+ /*limit_val=*/4>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
round_1, bits, idx_tbl);
} else {
HWY_IF_CONSTEXPR(taps_y_const > 0) {
ConvolveVerticalPass<taps_y_const, true, false, round_0_const,
- round_1_const>(
+ round_1_const, /*limit_val=*/4>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
round_const_y, round_1, bits, idx_tbl);
}
else {
if (taps_y == 2) {
- ConvolveVerticalPass<2, true, false, round_0_const,
- round_1_const>(
+ ConvolveVerticalPass<2, true, false, round_0_const, round_1_const,
+ /*limit_val=*/4>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
round_const_y, round_1, bits, idx_tbl);
} else if (taps_y == 4) {
- ConvolveVerticalPass<4, true, false, round_0_const,
- round_1_const>(
+ ConvolveVerticalPass<4, true, false, round_0_const, round_1_const,
+ /*limit_val=*/4>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
round_const_y, round_1, bits, idx_tbl);
} else if (taps_y == 6) {
- ConvolveVerticalPass<6, true, false, round_0_const,
- round_1_const>(
+ ConvolveVerticalPass<6, true, false, round_0_const, round_1_const,
+ /*limit_val=*/4>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
round_const_y, round_1, bits, idx_tbl);
} else if (taps_y == 8) {
- ConvolveVerticalPass<8, true, false, round_0_const,
- round_1_const>(
+ ConvolveVerticalPass<8, true, false, round_0_const, round_1_const,
+ /*limit_val=*/4>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
round_const_y, round_1, bits, idx_tbl);
}
}
}
} else {
- if (is_taps_y_12) {
- ConvolveVerticalPass<12, false, false, round_0_const, round_1_const>(
- strip_im_buf, dst + j, dst_stride, w, h, coeffs_v, round_const_y,
- round_1, bits, idx_tbl);
- } else {
- HWY_IF_CONSTEXPR(taps_y_const > 0) {
- ConvolveVerticalPass<taps_y_const, false, false, round_0_const,
- round_1_const>(
+ if (h == 2) {
+ if (is_taps_y_12) {
+ ConvolveVerticalPass<12, false, false, round_0_const, round_1_const,
+ /*limit_val=*/1>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
round_const_y, round_1, bits, idx_tbl);
- }
- else {
- if (taps_y == 2) {
- ConvolveVerticalPass<2, false, false, round_0_const,
- round_1_const>(
- strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
- round_const_y, round_1, bits, idx_tbl);
- } else if (taps_y == 4) {
- ConvolveVerticalPass<4, false, false, round_0_const,
- round_1_const>(
+ } else {
+ HWY_IF_CONSTEXPR(taps_y_const > 0) {
+ ConvolveVerticalPass<taps_y_const, false, false, round_0_const,
+ round_1_const, /*limit_val=*/1>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
round_const_y, round_1, bits, idx_tbl);
- } else if (taps_y == 6) {
- ConvolveVerticalPass<6, false, false, round_0_const,
- round_1_const>(
+ }
+ else {
+ if (taps_y == 2) {
+ ConvolveVerticalPass<2, false, false, round_0_const,
+ round_1_const, /*limit_val=*/1>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ } else if (taps_y == 4) {
+ ConvolveVerticalPass<4, false, false, round_0_const,
+ round_1_const, /*limit_val=*/1>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ } else if (taps_y == 6) {
+ ConvolveVerticalPass<6, false, false, round_0_const,
+ round_1_const, /*limit_val=*/1>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ } else if (taps_y == 8) {
+ ConvolveVerticalPass<8, false, false, round_0_const,
+ round_1_const, /*limit_val=*/1>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ }
+ }
+ }
+ } else if (h == 6) {
+ if (is_taps_y_12) {
+ ConvolveVerticalPass<12, false, false, round_0_const, round_1_const,
+ /*limit_val=*/3>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ } else {
+ HWY_IF_CONSTEXPR(taps_y_const > 0) {
+ ConvolveVerticalPass<taps_y_const, false, false, round_0_const,
+ round_1_const, /*limit_val=*/3>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
round_const_y, round_1, bits, idx_tbl);
- } else if (taps_y == 8) {
- ConvolveVerticalPass<8, false, false, round_0_const,
- round_1_const>(
+ }
+ else {
+ if (taps_y == 2) {
+ ConvolveVerticalPass<2, false, false, round_0_const,
+ round_1_const, /*limit_val=*/3>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ } else if (taps_y == 4) {
+ ConvolveVerticalPass<4, false, false, round_0_const,
+ round_1_const, /*limit_val=*/3>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ } else if (taps_y == 6) {
+ ConvolveVerticalPass<6, false, false, round_0_const,
+ round_1_const, /*limit_val=*/3>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ } else if (taps_y == 8) {
+ ConvolveVerticalPass<8, false, false, round_0_const,
+ round_1_const, /*limit_val=*/3>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ }
+ }
+ }
+ } else {
+ if (is_taps_y_12) {
+ ConvolveVerticalPass<12, false, false, round_0_const, round_1_const,
+ /*limit_val=*/2>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ } else {
+ HWY_IF_CONSTEXPR(taps_y_const > 0) {
+ ConvolveVerticalPass<taps_y_const, false, false, round_0_const,
+ round_1_const, /*limit_val=*/2>(
strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
round_const_y, round_1, bits, idx_tbl);
}
+ else {
+ if (taps_y == 2) {
+ ConvolveVerticalPass<2, false, false, round_0_const,
+ round_1_const, /*limit_val=*/2>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ } else if (taps_y == 4) {
+ ConvolveVerticalPass<4, false, false, round_0_const,
+ round_1_const, /*limit_val=*/2>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ } else if (taps_y == 6) {
+ ConvolveVerticalPass<6, false, false, round_0_const,
+ round_1_const, /*limit_val=*/2>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ } else if (taps_y == 8) {
+ ConvolveVerticalPass<8, false, false, round_0_const,
+ round_1_const, /*limit_val=*/2>(
+ strip_im_buf, dst + j, dst_stride, w, h, coeffs_v,
+ round_const_y, round_1, bits, idx_tbl);
+ }
+ }
}
}
}
diff --git a/test/av1_convolve_test.cc b/test/av1_convolve_test.cc
index d52460ddab..7697285773 100644
--- a/test/av1_convolve_test.cc
+++ b/test/av1_convolve_test.cc
@@ -1314,11 +1314,16 @@ class AV1Convolve2DTest : public AV1ConvolveTest<convolve_2d_func> {
height, filter_params_x, filter_params_y, sub_x, sub_y,
&conv_params1);
DECLARE_ALIGNED(32, uint8_t, test[MAX_SB_SQUARE]);
+ const int max_dst_offset = height * kOutputStride;
+ ASAN_POISON_MEMORY_REGION(test + max_dst_offset,
+ sizeof(test) - max_dst_offset);
ConvolveParams conv_params2 =
get_conv_params_no_round(0, 0, nullptr, 0, 0, 8);
GetParam().TestFunction()(input, input_stride, test, kOutputStride, width,
height, filter_params_x, filter_params_y, sub_x,
sub_y, &conv_params2);
+ ASAN_UNPOISON_MEMORY_REGION(test + max_dst_offset,
+ sizeof(test) - max_dst_offset);
ASAN_UNPOISON_MEMORY_REGION(input + max_input_offset + 1, 16);
ASAN_UNPOISON_MEMORY_REGION(input + min_input_offset - 16, 16);
AssertOutputBufferEq(reference, test, width, height);