Commit b3940a4a for libheif
commit b3940a4a5539e7c9e15f34de3c51e75d7fabd88f
Author: Dirk Farin <dirk.farin@gmail.com>
Date: Sun Sep 6 15:20:47 2026 +0200
Check every sequence frame against the encoder's bit depth
An encoder is opened once per sequence and configured from the first frame:
Encoder_HEVC / Encoder_AVC / Encoder_AVIF / Encoder_VVC::encode_sequence_frame()
call the plugin's start_sequence_encoding() only while the encoder is not
running yet. Every later frame goes straight to encode_sequence_frame(), where
the only bit depth check was the codec level set {8, 10, 12}, and nothing in
the sequence API requires the frames of a track to agree on a depth.
x265 is the worst case. It hands libx265 the plane pointers of the current
frame together with a pic->bitDepth taken from the first one, so a 10 bit first
frame followed by an 8 bit frame made libx265 walk a one byte per sample plane
at two bytes per sample. Valgrind reports the heap out-of-bounds read inside
x265_10bit::Encoder::encode(). libaom refused such a frame with an error of its
own that named no cause, while rav1e and SVT-AV1 quietly encoded the mismatch.
check_sequence_frame_bit_depth() now pins every frame to the depth the encoder
was opened with, in the x265, aom, SVT-AV1 and rav1e plugins. uvg266 needs no
such state: its depth is fixed at compile time, so passing UVG_BIT_DEPTH instead
of {8, 10, 12} to check_encoder_input_image() pins all frames at once. That also
closes the gap left by the uvg266 start check added in 6c167db5, which only ever
sees the first frame of a sequence.
Also correct the comment claiming that uvg_api_get() checks what the linked
build supports. Upstream it is 'return &uvg_8bit_api;' and never fails, and
kvz_api_get() is the same; only x265_api_get() answers the question at run time.
diff --git a/libheif/plugins/encoder_aom.cc b/libheif/plugins/encoder_aom.cc
index 4cad0b2e..eb642708 100644
--- a/libheif/plugins/encoder_aom.cc
+++ b/libheif/plugins/encoder_aom.cc
@@ -98,6 +98,9 @@ struct encoder_struct_aom
heif_chroma chroma = heif_chroma_420;
+ // bit depth the codec was initialized with, to check the later frames of a sequence against
+ int bit_depth = 8;
+
// --- input
bool alpha_quality_set = false;
@@ -1170,6 +1173,8 @@ static heif_error aom_start_sequence_encoding_intern(void* encoder_raw, const he
return err;
}
+ encoder->bit_depth = bpp_y;
+
aom_codec_err_t aom_error;
aom_error = aom_codec_control(&codec, AOME_SET_CPUUSED, encoder->cpu_used); CHECK_ERROR;
@@ -1267,6 +1272,14 @@ static heif_error aom_encode_sequence_frame(void* encoder_raw, const heif_image*
encoder_struct_aom* encoder = (encoder_struct_aom*) encoder_raw;
aom_codec_ctx_t& codec = encoder->codec;
+ // AOM_CODEC_USE_HIGHBITDEPTH was decided when the codec was initialized from the
+ // first frame of the sequence. libaom refuses a frame that disagrees with it, but
+ // with an error that says nothing about the cause.
+ input_error = check_sequence_frame_bit_depth(image, encoder->bit_depth);
+ if (input_error.code != heif_error_Ok) {
+ return input_error;
+ }
+
heif_error err;
const int source_width = heif_image_get_width(image, heif_channel_Y);
diff --git a/libheif/plugins/encoder_input_check.h b/libheif/plugins/encoder_input_check.h
index b83ab5a1..127fd762 100644
--- a/libheif/plugins/encoder_input_check.h
+++ b/libheif/plugins/encoder_input_check.h
@@ -54,10 +54,14 @@
* - JPEG 2000 signals a precision per component and genuinely supports it,
* which is why the OpenJPEG and OpenJPH plugins do not use this check.
*
- * 'supported_bit_depths' is the set the codec allows. It is a codec level
- * constraint and does not replace a plugin's own check against the encoder
- * library actually linked in (x265_api_get(), uvg_api_get() and friends), which
- * is what decides whether this particular build can do 10 or 12 bits.
+ * 'supported_bit_depths' is the set this plugin can actually encode. Where that
+ * is narrower than what the codec allows because of the encoder library, the
+ * plugin has to pass the narrower set: only x265 answers the question at run
+ * time, where x265_api_get() returns NULL for a depth the linked build does not
+ * provide. uvg_api_get() and kvz_api_get() look like they do the same, but both
+ * are 'return &..._8bit_api;' upstream and never fail, so the uvg266 and kvazaar
+ * builds are pinned at compile time by UVG_BIT_DEPTH / KVZ_BIT_DEPTH and those
+ * are what the two plugins pass here.
*
* TODO: this per-plugin check is a stopgap. What is really needed is a proper
* plugin API through which an encoder describes the input formats it accepts
@@ -125,4 +129,35 @@ static inline heif_error check_encoder_input_image(const heif_image* image,
"Encoder cannot encode images at this bit depth"};
}
+
+/*
+ * An encoder is opened once per sequence, in *_start_sequence_encoding(), and is
+ * configured from the first frame. Encoder_HEVC / Encoder_AVC / Encoder_AVIF /
+ * Encoder_VVC::encode_sequence_frame() call it only while the encoder is not
+ * running yet, so every later frame goes straight to *_encode_sequence_frame().
+ * A plugin that latched a bit depth there and applies it to the planes of a later
+ * frame walks a one byte per sample plane at two bytes per sample: x265, for
+ * example, hands the plane pointers of the current frame to libx265 together with
+ * pic->bitDepth taken from the first frame, and reads past the end of them.
+ *
+ * So a plugin that keeps the bit depth across frames has to check every frame
+ * against that configuration and not only against the codec level set above.
+ * Plugins whose depth is fixed at compile time need nothing extra: passing that
+ * constant to check_encoder_input_image() already pins all frames to one value.
+ *
+ * Call this after check_encoder_input_image(), which has already established that
+ * the luma channel exists and that the chroma channels have the same depth.
+ */
+static inline heif_error check_sequence_frame_bit_depth(const heif_image* image,
+ int configured_bit_depth)
+{
+ if (heif_image_get_bits_per_pixel_range(image, heif_channel_Y) != configured_bit_depth) {
+ return heif_error{heif_error_Encoder_plugin_error,
+ heif_suberror_Unsupported_bit_depth,
+ "All frames of a sequence must have the bit depth of the first frame"};
+ }
+
+ return heif_error_ok;
+}
+
#endif // LIBHEIF_ENCODER_INPUT_CHECK_H
diff --git a/libheif/plugins/encoder_rav1e.cc b/libheif/plugins/encoder_rav1e.cc
index 2a3205c8..f834e525 100644
--- a/libheif/plugins/encoder_rav1e.cc
+++ b/libheif/plugins/encoder_rav1e.cc
@@ -61,6 +61,9 @@ struct encoder_struct_rav1e
RaContext* rav1eContextRaw = nullptr;
uint8_t yShift = 0;
+ // bit depth the context was created with, to check the later frames of a sequence against
+ int bit_depth = 8;
+
// --- output
struct Packet
@@ -659,6 +662,8 @@ heif_error rav1e_start_sequence_encoding_intern(void* encoder_raw, const heif_im
return heif_error_codec_library_error;
}
+ encoder->bit_depth = bitDepth;
+
return {};
}
@@ -686,6 +691,14 @@ heif_error rav1e_encode_sequence_frame(void* encoder_raw, const heif_image* imag
auto* encoder = (encoder_struct_rav1e*) encoder_raw;
auto& rav1eContext = encoder->rav1eContextRaw;
+ // rav1e_frame_new() below builds a frame for the pixel format the context was
+ // created with, from the first frame of the sequence, while byteWidth is this
+ // frame's.
+ input_error = check_sequence_frame_bit_depth(image, encoder->bit_depth);
+ if (input_error.code != heif_error_Ok) {
+ return input_error;
+ }
+
int bitDepth = heif_image_get_bits_per_pixel_range(image, heif_channel_Y);
int yShift = encoder->yShift;
diff --git a/libheif/plugins/encoder_svt.cc b/libheif/plugins/encoder_svt.cc
index 04d4d79b..4ce65970 100644
--- a/libheif/plugins/encoder_svt.cc
+++ b/libheif/plugins/encoder_svt.cc
@@ -89,7 +89,7 @@ struct encoder_struct_svt
// --- Encoder
EbComponentType* svt_encoder = nullptr;
- EbSvtAv1EncConfiguration svt_config;
+ EbSvtAv1EncConfiguration svt_config{}; // value-initialized: svt_encode_sequence_frame() reads encoder_bit_depth from it
EbBufferHeaderType input_buffer;
bool still_image_mode = false;
@@ -1035,6 +1035,13 @@ static heif_error svt_encode_sequence_frame(void* encoder_raw, const heif_image*
EbComponentType*& svt_encoder = encoder->svt_encoder;
EbErrorType res = EB_ErrorNone;
+ // svt_config.encoder_bit_depth was taken from the first frame of the sequence,
+ // while the plane pointers handed to SVT below are this frame's.
+ input_error = check_sequence_frame_bit_depth(image, encoder->svt_config.encoder_bit_depth);
+ if (input_error.code != heif_error_Ok) {
+ return input_error;
+ }
+
int w = heif_image_get_width(image, heif_channel_Y);
int h = heif_image_get_height(image, heif_channel_Y);
const heif_chroma chroma = heif_image_get_chroma_format(image);
diff --git a/libheif/plugins/encoder_uvg266.cc b/libheif/plugins/encoder_uvg266.cc
index 407439d7..f6559ff0 100644
--- a/libheif/plugins/encoder_uvg266.cc
+++ b/libheif/plugins/encoder_uvg266.cc
@@ -660,10 +660,12 @@ static heif_error uvg266_start_sequence_encoding_intern(void* encoder_raw, const
static heif_error uvg266_encode_sequence_frame(void* encoder_raw, const heif_image* image,
uintptr_t framenr)
{
- // VVC signals one bit depth for all planes. Whether this build of uvg266
- // supports the depth is checked separately via uvg_api_get().
+ // VVC signals one bit depth for all planes, and uvg266 is built for a single
+ // depth: uvg_pixel follows UVG_BIT_DEPTH. This has to be checked here and not
+ // only in uvg266_start_sequence_encoding_intern(), because that one runs for
+ // the first frame of a sequence only.
heif_error input_error = check_encoder_input_image(image, /*supports_monochrome=*/true,
- {8, 10, 12});
+ {UVG_BIT_DEPTH});
if (input_error.code != heif_error_Ok) {
return input_error;
}
diff --git a/libheif/plugins/encoder_x265.cc b/libheif/plugins/encoder_x265.cc
index 8a8561c5..87e2a03a 100644
--- a/libheif/plugins/encoder_x265.cc
+++ b/libheif/plugins/encoder_x265.cc
@@ -1102,6 +1102,16 @@ static heif_error x265_start_sequence_encoding(void* encoder_raw, const heif_ima
static heif_error x265_encode_sequence_frame(void* encoder_raw, const heif_image* image,
uintptr_t frame_nr)
{
+ encoder_struct_x265* encoder = (encoder_struct_x265*) encoder_raw;
+
+ if (!encoder->api) {
+ return {
+ heif_error_Usage_error,
+ heif_suberror_Unspecified,
+ "called plugin encode_sequence_frame() without start_sequence_encoding()"
+ };
+ }
+
// HEVC can signal different luma and chroma bit depths, but x265 has a
// single internal bit depth and cannot produce such a stream. Whether this
// build of libx265 has 10 or 12 bit support is checked separately via
@@ -1112,14 +1122,12 @@ static heif_error x265_encode_sequence_frame(void* encoder_raw, const heif_image
return input_error;
}
- encoder_struct_x265* encoder = (encoder_struct_x265*) encoder_raw;
-
- if (!encoder->api) {
- return {
- heif_error_Usage_error,
- heif_suberror_Unspecified,
- "called plugin encode_sequence_frame() without start_sequence_encoding()"
- };
+ // pic->bitDepth below is the depth the encoder was opened with, while the plane
+ // pointers are this frame's. A deeper first frame would make libx265 read the
+ // planes of a shallower later frame at two bytes per sample.
+ input_error = check_sequence_frame_bit_depth(image, encoder->bit_depth);
+ if (input_error.code != heif_error_Ok) {
+ return input_error;
}
const x265_api* api = encoder->api;
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 5d7b4b6a..6f9d9913 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -97,6 +97,7 @@ find_package(Threads REQUIRED)
target_link_libraries(alpha_cycle_deadlock PRIVATE Threads::Threads)
target_link_libraries(parallel_grid_deadlock PRIVATE Threads::Threads)
add_libheif_test(region)
+add_libheif_test(sequence_mixed_bit_depth)
add_libheif_test(sequence_no_track)
add_libheif_test(sequence_null_options)
add_libheif_test(sequence_timing_overflow)
diff --git a/tests/sequence_mixed_bit_depth.cc b/tests/sequence_mixed_bit_depth.cc
new file mode 100644
index 00000000..c0f65c39
--- /dev/null
+++ b/tests/sequence_mixed_bit_depth.cc
@@ -0,0 +1,237 @@
+/*
+ libheif unit tests
+
+ MIT License
+
+ Copyright (c) 2026 Dirk Farin <dirk.farin@gmail.com>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+// An encoder is opened once per sequence and configured from the first frame:
+// Encoder_HEVC / Encoder_AVC / Encoder_AVIF / Encoder_VVC::encode_sequence_frame()
+// call the plugin's start_sequence_encoding() only while the encoder is not
+// running yet. Every later frame goes straight to encode_sequence_frame(), and
+// nothing in the sequence API requires the frames to agree on a bit depth.
+//
+// The plugins latch the depth at that point. x265 is the worst case: it hands
+// libx265 the plane pointers of the current frame together with a pic->bitDepth
+// taken from the first one, so a 10 bit first frame followed by an 8 bit frame
+// made libx265 walk a one byte per sample plane at two bytes per sample. That is
+// a heap out-of-bounds read, and it reproduces under valgrind on the unfixed code
+// as an "Invalid read" inside x265_10bit::Encoder::encode(). aom refused such a
+// frame with an error of its own that named no cause, and rav1e and SVT-AV1
+// quietly encoded the mismatch.
+//
+// The fix is check_sequence_frame_bit_depth() in the plugins, so every frame of
+// a sequence has to carry the depth the encoder was opened with. This test pins
+// both directions of the mismatch, and that a uniform sequence still encodes.
+
+#include "catch_amalgamated.hpp"
+#include "libheif/heif.h"
+#include "libheif/heif_sequences.h"
+#include "test_utils.h"
+
+#include <cstdint>
+#include <cstring>
+
+namespace {
+
+// Large enough that the over-read of the unfixed code runs past the end of the
+// plane allocation instead of staying inside its stride padding.
+constexpr uint32_t WIDTH = 512;
+constexpr uint32_t HEIGHT = 512;
+
+heif_image* make_image(int bit_depth)
+{
+ heif_image* img = nullptr;
+ REQUIRE(heif_image_create(WIDTH, HEIGHT, heif_colorspace_YCbCr, heif_chroma_420, &img).code == heif_error_Ok);
+ REQUIRE(img != nullptr);
+
+ REQUIRE(heif_image_add_plane(img, heif_channel_Y, WIDTH, HEIGHT, bit_depth).code == heif_error_Ok);
+ REQUIRE(heif_image_add_plane(img, heif_channel_Cb, WIDTH / 2, HEIGHT / 2, bit_depth).code == heif_error_Ok);
+ REQUIRE(heif_image_add_plane(img, heif_channel_Cr, WIDTH / 2, HEIGHT / 2, bit_depth).code == heif_error_Ok);
+
+ for (heif_channel channel : {heif_channel_Y, heif_channel_Cb, heif_channel_Cr}) {
+ uint32_t h = (channel == heif_channel_Y) ? HEIGHT : HEIGHT / 2;
+ uint32_t w = (channel == heif_channel_Y) ? WIDTH : WIDTH / 2;
+
+ size_t stride = 0;
+ uint8_t* p = heif_image_get_plane2(img, channel, &stride);
+ REQUIRE(p != nullptr);
+
+ // Mid grey, so that a frame read at the wrong sample width is visibly wrong
+ // rather than accidentally plausible.
+ uint16_t value = static_cast<uint16_t>(1 << (bit_depth - 1));
+
+ for (uint32_t y = 0; y < h; y++) {
+ if (bit_depth > 8) {
+ auto* row = reinterpret_cast<uint16_t*>(p + y * stride);
+ for (uint32_t x = 0; x < w; x++) {
+ row[x] = value;
+ }
+ }
+ else {
+ memset(p + y * stride, static_cast<int>(value), w);
+ }
+ }
+ }
+
+ heif_image_set_duration(img, 1);
+
+ return img;
+}
+
+struct TwoFrameResult
+{
+ // False when this build cannot encode the first depth at all (a libx265 without
+ // high bit depth support, for instance). The sequence then never reached a state
+ // in which two frames could disagree, and there is nothing to assert.
+ bool first_frame_encoded = false;
+
+ heif_error second_frame_error = heif_error{heif_error_Ok, heif_suberror_Unspecified, nullptr};
+};
+
+// Encode 'first_depth' and then 'second_depth' into one track and report what the
+// second frame returned.
+TwoFrameResult encode_two_frames(heif_compression_format format, int first_depth, int second_depth)
+{
+ heif_context* ctx = heif_context_alloc();
+ REQUIRE(ctx != nullptr);
+
+ heif_encoder* encoder = nullptr;
+ REQUIRE(heif_context_get_encoder_for_format(ctx, format, &encoder).code == heif_error_Ok);
+ REQUIRE(encoder != nullptr);
+
+ heif_track* track = nullptr;
+ REQUIRE(heif_context_add_visual_sequence_track(ctx, static_cast<uint16_t>(WIDTH), static_cast<uint16_t>(HEIGHT),
+ heif_track_type_video, nullptr, nullptr, &track).code == heif_error_Ok);
+ REQUIRE(track != nullptr);
+
+ TwoFrameResult result;
+
+ heif_image* first = make_image(first_depth);
+ heif_error err = heif_track_encode_sequence_image(track, first, encoder, nullptr);
+ heif_image_release(first);
+
+ result.first_frame_encoded = (err.code == heif_error_Ok);
+
+ if (result.first_frame_encoded) {
+ heif_image* second = make_image(second_depth);
+ result.second_frame_error = heif_track_encode_sequence_image(track, second, encoder, nullptr);
+ heif_image_release(second);
+ }
+
+ heif_encoder_release(encoder);
+ heif_context_free(ctx);
+
+ return result;
+}
+
+void require_mismatch_refused(heif_compression_format format, int first_depth, int second_depth)
+{
+ TwoFrameResult result = encode_two_frames(format, first_depth, second_depth);
+
+ if (!result.first_frame_encoded) {
+ return;
+ }
+
+ const heif_error& err = result.second_frame_error;
+
+ INFO("second frame error (" << err.code << "/" << err.subcode << "): "
+ << (err.message ? err.message : "(null)"));
+
+ // Silently succeeding is the failure mode of the unfixed x265 plugin: it read
+ // past the end of the second frame's planes and encoded whatever it found.
+ REQUIRE(err.code != heif_error_Ok);
+
+ // An encoder may bail out for a reason of its own, but when it is our check
+ // that fires, it has to report the bit depth as the reason.
+ if (err.code == heif_error_Encoder_plugin_error) {
+ REQUIRE(err.subcode == heif_suberror_Unsupported_bit_depth);
+ }
+}
+
+void require_uniform_accepted(heif_compression_format format, int depth)
+{
+ TwoFrameResult result = encode_two_frames(format, depth, depth);
+
+ REQUIRE(result.first_frame_encoded);
+
+ const heif_error& err = result.second_frame_error;
+
+ INFO("second frame error (" << err.code << "/" << err.subcode << "): "
+ << (err.message ? err.message : "(null)"));
+ REQUIRE(err.code == heif_error_Ok);
+}
+
+} // namespace
+
+// One section per codec, so that a failure in one does not stop the other from
+// running: the AV1 encoders report the mismatch as an error of their own, while
+// it is the HEVC leg that reads out of bounds without the fix.
+TEST_CASE("a sequence frame must carry the bit depth of the first frame")
+{
+ SECTION("AV1")
+ {
+ if (!heif_have_encoder_for_format(heif_compression_AV1)) {
+ SKIP("Skipping because no AV1 encoder is compiled.");
+ }
+
+ // 10 bit first: this is the direction that made x265 read past the end of the
+ // 8 bit planes of the second frame.
+ require_mismatch_refused(heif_compression_AV1, 10, 8);
+
+ // 8 bit first: the mirror case, where the second frame's planes are wider than
+ // what the encoder was opened for.
+ require_mismatch_refused(heif_compression_AV1, 8, 10);
+ }
+
+ SECTION("HEVC")
+ {
+ if (!heif_have_encoder_for_format(heif_compression_HEVC)) {
+ SKIP("Skipping because no HEVC encoder is compiled.");
+ }
+
+ require_mismatch_refused(heif_compression_HEVC, 10, 8);
+ require_mismatch_refused(heif_compression_HEVC, 8, 10);
+ }
+}
+
+TEST_CASE("a sequence of frames with one bit depth still encodes")
+{
+ // The check must not reject what it is supposed to let through.
+ SECTION("AV1")
+ {
+ if (!heif_have_encoder_for_format(heif_compression_AV1)) {
+ SKIP("Skipping because no AV1 encoder is compiled.");
+ }
+
+ require_uniform_accepted(heif_compression_AV1, 8);
+ }
+
+ SECTION("HEVC")
+ {
+ if (!heif_have_encoder_for_format(heif_compression_HEVC)) {
+ SKIP("Skipping because no HEVC encoder is compiled.");
+ }
+
+ require_uniform_accepted(heif_compression_HEVC, 8);
+ }
+}