Commit 175070e28a for aom
commit 175070e28ab5d2dc5bf77338d7c7bc3044222b44
Author: Christopher Trask <ctrask@google.com>
Date: Wed Sep 16 19:45:16 2026 +0000
av1: Fix temporal filter SEGV on cancellation
Add defensive checks in av1_tf_info_filtering and encode_strategy.c to
ensure frames exist in the lookahead buffer before attempting temporal
filtering. This prevents a NULL pointer dereference when the encoder is
forced to flush early during second-pass encoding.
Bug: b/465818130
Change-Id: I51e517e46434059196999ce593aac4f3b632326f
diff --git a/av1/encoder/encode_strategy.c b/av1/encoder/encode_strategy.c
index 6f9715c2df..3cf86a2c6e 100644
--- a/av1/encoder/encode_strategy.c
+++ b/av1/encoder/encode_strategy.c
@@ -835,14 +835,17 @@ static int denoise_and_encode(AV1_COMP *const cpi, uint8_t *const dest,
// Right now, we are still using tf_buf_second_arf due to
// implementation complexity.
// TODO(angiebird): Reuse tf_info->tf_buf here.
- av1_temporal_filter(cpi, arf_src_index, cpi->gf_frame_index, &frame_diff,
- tf_buf_second_arf);
- show_existing_alt_ref =
- av1_check_show_filtered_frame(tf_buf_second_arf, &frame_diff, q_index,
- cm->seq_params->bit_depth, 1, 1);
- if (show_existing_alt_ref) {
- aom_extend_frame_borders(tf_buf_second_arf, av1_num_planes(cm));
- frame_input->source = tf_buf_second_arf;
+ if (av1_lookahead_peek(cpi->ppi->lookahead, arf_src_index,
+ cpi->compressor_stage)) {
+ av1_temporal_filter(cpi, arf_src_index, cpi->gf_frame_index,
+ &frame_diff, tf_buf_second_arf);
+ show_existing_alt_ref = av1_check_show_filtered_frame(
+ tf_buf_second_arf, &frame_diff, q_index, cm->seq_params->bit_depth,
+ 1, 1);
+ if (show_existing_alt_ref) {
+ aom_extend_frame_borders(tf_buf_second_arf, av1_num_planes(cm));
+ frame_input->source = tf_buf_second_arf;
+ }
}
// Currently INTNL_ARF_UPDATE only do show_existing.
cpi->common.showable_frame |= 1;
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index 97be8fc2b8..9498caf14d 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -1724,6 +1724,10 @@ void av1_tf_info_filtering(TEMPORAL_FILTER_INFO *tf_info, AV1_COMP *cpi,
int buf_idx = gf_group->frame_type[gf_index] == INTER_FRAME;
int lookahead_idx = gf_group->arf_src_offset[gf_index] +
gf_group->cur_frame_idx[gf_index];
+ if (!av1_lookahead_peek(cpi->ppi->lookahead, lookahead_idx,
+ cpi->compressor_stage)) {
+ continue;
+ }
// This function is designed to be called multiple times after
// av1_tf_info_reset(). It will only generate the filtered frame that does
// not exist yet.
diff --git a/test/cancellation_test.cc b/test/cancellation_test.cc
new file mode 100644
index 0000000000..e0a6f991e9
--- /dev/null
+++ b/test/cancellation_test.cc
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2026, Alliance for Open Media. All rights reserved.
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+#include <memory>
+
+#include "gtest/gtest.h"
+#include "test/codec_factory.h"
+#include "test/encode_test_driver.h"
+#include "test/util.h"
+#include "test/video_source.h"
+
+namespace {
+
+class CancellableVideoSource : public ::libaom_test::VideoSource {
+ public:
+ explicit CancellableVideoSource(::libaom_test::VideoSource *source)
+ : source_(source), cancel_after_(-1), current_pass_(0) {}
+
+ void Begin() override {
+ source_->Begin();
+ current_pass_++;
+ }
+
+ void Next() override { source_->Next(); }
+
+ aom_image_t *img() const override {
+ if (current_pass_ == 2 && cancel_after_ >= 0 &&
+ static_cast<int>(source_->frame()) >= cancel_after_) {
+ return nullptr;
+ }
+ return source_->img();
+ }
+
+ aom_codec_pts_t pts() const override { return source_->pts(); }
+ unsigned long duration() const override { return source_->duration(); }
+ aom_rational_t timebase() const override { return source_->timebase(); }
+ unsigned int frame() const override { return source_->frame(); }
+ unsigned int limit() const override { return source_->limit(); }
+
+ void set_cancel_after(int frame) { cancel_after_ = frame; }
+
+ private:
+ ::libaom_test::VideoSource *source_;
+ int cancel_after_;
+ int current_pass_;
+};
+
+class CancellationTest
+ : public ::libaom_test::EncoderTest,
+ public ::libaom_test::CodecTestWithParam< ::libaom_test::TestMode> {
+ protected:
+ CancellationTest()
+ : EncoderTest(GET_PARAM(0)), tile_columns_(0), tile_rows_(0) {}
+ ~CancellationTest() override = default;
+
+ void SetUp() override { InitializeConfig(GET_PARAM(1)); }
+
+ void PreEncodeFrameHook(::libaom_test::VideoSource *video,
+ ::libaom_test::Encoder *encoder) override {
+ if (video->frame() == 0) {
+ encoder->Control(AOME_SET_CPUUSED, 4); // speed 4
+ encoder->Control(AV1E_SET_TILE_COLUMNS, tile_columns_);
+ encoder->Control(AV1E_SET_TILE_ROWS, tile_rows_);
+ }
+ }
+
+ void DoSimulateCancellationPass2(int threads, int tile_columns,
+ int tile_rows) {
+ cfg_.g_threads = threads;
+ tile_columns_ = tile_columns;
+ tile_rows_ = tile_rows;
+
+ ::libaom_test::RandomVideoSource raw_video;
+ raw_video.SetSize(256, 256);
+ raw_video.set_limit(40); // 40 frames total
+
+ CancellableVideoSource video(&raw_video);
+ video.set_cancel_after(20); // Cancel after 20 frames in pass 2
+
+ // This should not crash
+ ASSERT_NO_FATAL_FAILURE(RunLoop(&video));
+ }
+
+ private:
+ int tile_columns_;
+ int tile_rows_;
+};
+
+TEST_P(CancellationTest, SimulateCancellationPass2) {
+ DoSimulateCancellationPass2(/*threads=*/1, /*tile_columns=*/0,
+ /*tile_rows=*/0);
+}
+
+TEST_P(CancellationTest, SimulateCancellationPass2Threads4) {
+ DoSimulateCancellationPass2(/*threads=*/4, /*tile_columns=*/1,
+ /*tile_rows=*/1);
+}
+
+// Test 2-pass good quality
+AV1_INSTANTIATE_TEST_SUITE(CancellationTest,
+ ::testing::Values(::libaom_test::kTwoPassGood));
+
+} // namespace
diff --git a/test/test.cmake b/test/test.cmake
index 84ba9182f5..655268a6e7 100644
--- a/test/test.cmake
+++ b/test/test.cmake
@@ -65,6 +65,7 @@ list(APPEND AOM_UNIT_TEST_ENCODER_SOURCES
"${AOM_ROOT}/test/av1_external_partition_test.cc"
"${AOM_ROOT}/test/avif_progressive_test.cc"
"${AOM_ROOT}/test/borders_test.cc"
+ "${AOM_ROOT}/test/cancellation_test.cc"
"${AOM_ROOT}/test/cpu_speed_test.cc"
"${AOM_ROOT}/test/cpu_used_firstpass_test.cc"
"${AOM_ROOT}/test/datarate_test.cc"
@@ -126,6 +127,7 @@ if(CONFIG_REALTIME_ONLY)
"${AOM_ROOT}/test/av1_external_partition_test.cc"
"${AOM_ROOT}/test/avif_progressive_test.cc"
"${AOM_ROOT}/test/borders_test.cc"
+ "${AOM_ROOT}/test/cancellation_test.cc"
"${AOM_ROOT}/test/cpu_speed_test.cc"
"${AOM_ROOT}/test/cpu_used_firstpass_test.cc"
"${AOM_ROOT}/test/deltaq_mode_test.cc"