Commit 795523d8 for libheif
commit 795523d844b7b1f0818be389a1ebc9ce3ee8a547
Author: Dirk Farin <dirk.farin@gmail.com>
Date: Tue Aug 25 21:18:05 2026 +0200
limit the maximum x265 image size
diff --git a/SECURITY.md b/SECURITY.md
index 132f3d6f..9d96a76e 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -31,6 +31,16 @@ exceptionally hard to implement securely:
* **Multiple codecs behind one container.** The container makes claims (dimensions, chroma
format, bit depth, color information) that the embedded HEVC, AV1, JPEG, JPEG 2000 or VVC
bitstream may contradict. libheif has to reconcile both before any pixel buffer is touched.
+* **Many codec libraries.** libheif supports more than a dozen decoder and encoder backends
+ (libde265, x265, kvazaar, libaom, dav1d, rav1e, SVT-AV1, vvdec, vvenc, uvg266, openh264,
+ x264, ffmpeg, libjpeg, OpenJPEG, OpenJPH), in whatever versions the distributions ship.
+ Bugs and undocumented limits in these libraries surface through libheif, because libheif is
+ the component that hands them the untrusted data and that is named in the bug report.
+ Typical cases are integer overflows inside a codec at very large image sizes, crashes on
+ inputs that are valid for the container but exceed codec-internal limits, and frame buffers
+ allocated inside the codec that are outside libheif's memory accounting. libheif has to
+ anticipate these and compensate with its own checks and limits before data is passed to a
+ codec and after results come back, for code it does not control.
libheif validates all of this, is fuzzed continuously, and uses configurable security limits
to bound memory and CPU use. Still, the number of feature combinations is large, and as
diff --git a/libheif/plugins/encoder_x265.cc b/libheif/plugins/encoder_x265.cc
index cf9c74f9..97cd6e0d 100644
--- a/libheif/plugins/encoder_x265.cc
+++ b/libheif/plugins/encoder_x265.cc
@@ -1025,12 +1025,32 @@ static heif_error x265_start_sequence_encoding_intern(void* encoder_raw, const h
param->sourceWidth = rounded_size(param->sourceWidth);
param->sourceHeight = rounded_size(param->sourceHeight);
+ // x265 sizes some of its internal picture buffers with 32-bit arithmetic and, up to at least
+ // v3.5, neither validates the picture size nor the result of these allocations. Pictures of
+ // roughly 700 Mpixel or more make it crash in one of its worker threads (GHSA-2c3g-p585-8rpq).
+ // Newer x265 versions refuse pictures above the HEVC Level 7.2 maximum of 142,606,336 luma
+ // samples (16384x8704) in x265_check_params() unless non-conformance is explicitly allowed.
+ // Apply the same limit here, so that the behavior is the same with all x265 versions and the
+ // encoder is never opened with a picture it cannot handle.
+ const uint64_t max_luma_samples = 142606336;
+ if (static_cast<uint64_t>(param->sourceWidth) * static_cast<uint64_t>(param->sourceHeight) > max_luma_samples) {
+ return {heif_error_Encoding_error,
+ heif_suberror_Encoder_encoding,
+ "Image too large for x265: at most 142,606,336 luma samples (HEVC Level 7.2) are supported"};
+ }
+
param->fpsNum = framerate_num;
param->fpsDenom = framerate_denom;
encoder->bit_depth = bit_depth;
encoder->encoder = api->encoder_open(param);
+ if (encoder->encoder == nullptr) {
+ // x265 rejected the parameters (e.g. newer versions refuse oversized pictures).
+ return {heif_error_Encoding_error,
+ heif_suberror_Encoder_initialization,
+ "x265 encoder could not be opened with the given parameters"};
+ }
if (image_sequence) {
x265_nal* nals = nullptr;
diff --git a/tests/conversion.cc b/tests/conversion.cc
index 6c1e5b4c..fa970b66 100644
--- a/tests/conversion.cc
+++ b/tests/conversion.cc
@@ -894,3 +894,76 @@ TEST_CASE("Mismatched alpha bit depth - conversion correctness") {
CHECK(p[3] == 200); // A (10-bit 800 >> 2 = 200)
}
}
+
+
+// Regression test for GHSA-2c3g-p585-8rpq. Op_RGB24_32_to_YCbCr (like the other conversion
+// operations) used to compute the row offset y * stride in 32-bit int, which overflows as soon
+// as a plane exceeds 2 GB and made the conversion read from a wild address. All internal
+// strides are size_t since v1.19.6. Reproducing the overflow inherently needs a plane of more
+// than 2 GB (about 5 GB in total for this test), so the test is hidden from the default run.
+// Run it explicitly with:
+// ./tests/conversion "[large-memory]"
+TEST_CASE("RGB24 to YCbCr conversion with planes larger than 2 GB", "[.large-memory]")
+{
+ // Same size as the advisory PoC. The row stride is about 98 KB, so y * stride exceeds
+ // INT32_MAX for every row beyond roughly 21850.
+ const uint32_t w = 32767;
+ const uint32_t h = 32767;
+
+ // Creates an interleaved RGB image with a red first row and a blue last row.
+ auto make_image = [](uint32_t width, uint32_t height) {
+ auto img = std::make_shared<HeifPixelImage>();
+ img->create(width, height, heif_colorspace_RGB, heif_chroma_interleaved_RGB);
+ auto err = img->add_channel(heif_channel_interleaved, width, height, 8, nullptr);
+ REQUIRE(!err);
+
+ size_t stride;
+ uint8_t* first = img->get_channel_memory(heif_channel_interleaved, &stride);
+ uint8_t* last = first + static_cast<size_t>(height - 1) * stride;
+ for (uint32_t x = 0; x < width; x++) {
+ first[3 * x + 0] = 255;
+ first[3 * x + 1] = 0;
+ first[3 * x + 2] = 0;
+ last[3 * x + 0] = 0;
+ last[3 * x + 1] = 0;
+ last[3 * x + 2] = 255;
+ }
+ return img;
+ };
+
+ // Force nearest-neighbor chroma downsampling: this selects the single-step, per-pixel
+ // Op_RGB24_32_to_YCbCr from the advisory. The default (sharp yuv, if libsharpyuv is
+ // available) adjusts luma based on the neighboring rows, which are left uninitialized here.
+ heif_color_conversion_options options;
+ heif_color_conversion_options_set_defaults(&options);
+ options.preferred_chroma_downsampling_algorithm = heif_chroma_downsampling_nearest_neighbor;
+ options.only_use_preferred_chroma_algorithm = true;
+
+ auto convert = [&](const std::shared_ptr<HeifPixelImage>& img) {
+ auto result = convert_colorspace(img, heif_colorspace_YCbCr, heif_chroma_420,
+ nclx_profile::defaults(), 8, options, nullptr,
+ heif_get_disabled_security_limits());
+ REQUIRE(result);
+ return *result;
+ };
+
+ auto big = make_image(w, h);
+ size_t in_stride;
+ big->get_channel_memory(heif_channel_interleaved, &in_stride);
+ REQUIRE(static_cast<uint64_t>(in_stride) * (h - 1) > INT32_MAX); // the last row is only reachable with 64-bit offsets
+
+ auto big_out = convert(big);
+ big.reset();
+ auto small_out = convert(make_image(2, 2));
+
+ // The first and the last row of the large image must convert to the same luma values as
+ // the two rows of the small reference image.
+ size_t big_stride, small_stride;
+ const uint8_t* big_y = big_out->get_channel_memory(heif_channel_Y, &big_stride);
+ const uint8_t* small_y = small_out->get_channel_memory(heif_channel_Y, &small_stride);
+ REQUIRE(big_y[0] == small_y[0]);
+ REQUIRE(big_y[w - 1] == small_y[0]);
+ REQUIRE(big_y[static_cast<size_t>(h - 1) * big_stride] == small_y[small_stride]);
+ REQUIRE(big_y[static_cast<size_t>(h - 1) * big_stride + (w - 1)] == small_y[small_stride]);
+ REQUIRE(small_y[0] != small_y[small_stride]);
+}
diff --git a/tests/encode.cc b/tests/encode.cc
index 0d8ab553..c90f1c4b 100644
--- a/tests/encode.cc
+++ b/tests/encode.cc
@@ -156,3 +156,44 @@ TEST_CASE( "ispe odd size", "[heif_context]" ) {
test_ispe_size(heif_compression_AV1, heif_orientation_rotate_90_cw, 121,99, 121,99);
test_ispe_size(heif_compression_AV1, heif_orientation_rotate_90_cw, 120,100, 120,100);
}
+
+
+TEST_CASE("x265 rejects oversized images", "[heif_encoder]") {
+ // Regression test for GHSA-2c3g-p585-8rpq. x265 (up to at least v3.5) neither checks the
+ // picture size nor its internal allocation results and crashes in a worker thread for
+ // pictures of roughly 700 Mpixel or more. The x265 plugin therefore rejects pictures above
+ // the HEVC Level 7.2 maximum of 142,606,336 luma samples (16384x8704), which is the same
+ // limit newer x265 versions enforce themselves.
+
+ const heif_encoder_descriptor* descriptor = nullptr;
+ int n = heif_get_encoder_descriptors(heif_compression_HEVC, "x265", &descriptor, 1);
+ if (n == 0) {
+ SKIP("x265 encoder not available, skipping test");
+ }
+
+ heif_context* ctx = heif_context_alloc();
+ heif_encoder* enc = nullptr;
+ heif_error err = heif_context_get_encoder(ctx, descriptor, &enc);
+ REQUIRE(err.code == heif_error_Ok);
+
+ // One row more than Level 7.2 allows. A single 8-bit luma plane keeps this at about 142 MB.
+ const int w = 16384;
+ const int h = 8705;
+ heif_image* img = nullptr;
+ err = heif_image_create(w, h, heif_colorspace_monochrome, heif_chroma_monochrome, &img);
+ REQUIRE(err.code == heif_error_Ok);
+ err = heif_image_add_plane(img, heif_channel_Y, w, h, 8);
+ REQUIRE(err.code == heif_error_Ok);
+
+ heif_image_handle* handle = nullptr;
+ err = heif_context_encode_image(ctx, img, enc, nullptr, &handle);
+ REQUIRE(err.code == heif_error_Encoding_error);
+ REQUIRE(err.subcode == heif_suberror_Encoder_encoding);
+
+ if (handle) {
+ heif_image_handle_release(handle);
+ }
+ heif_image_release(img);
+ heif_encoder_release(enc);
+ heif_context_free(ctx);
+}