Commit 8a13bb23 for libheif
commit 8a13bb23f1208d034cfb76c4f7b6d3aa3e0b8fe7
Author: Dirk Farin <dirk.farin@gmail.com>
Date: Wed Aug 26 01:07:24 2026 +0200
hdr_sdr: reject out-of-range output bit depth in Op_to_hdr_planes (GHSA-8857-r8x5-7499)
Op_to_hdr_planes widens an 8-bit input to a higher bit depth using the
bit-replication identity out = (in << (m-8)) | (in >> (16-m)). That identity
only holds for target depths m in (8, 16]. For m > 16 the right shift exponent
(16-m) becomes negative, which is undefined behavior (UBSan: "shift exponent is
negative"), and m > 16 also does not fit the uint16_t output plane.
Guard both entry points: state_after_conversion() only offers the operation for
8 < m <= 16, and convert_colorspace() returns Unsupported_color_conversion for
any unsupported bit-depth combination instead of performing the shift (defense
in depth for direct callers).
No public API or untrusted-input path can drive the output depth above 16
(decode/encode pass the input depth or 0, and the RRGGBB path is clamped to 10
bits), so this is a hardening fix rather than a remotely reachable issue.
Adds a regression test covering the supported widenings and the rejected
out-of-range depths.
diff --git a/libheif/color-conversion/hdr_sdr.cc b/libheif/color-conversion/hdr_sdr.cc
index 8a6fda5a..ca70ef24 100644
--- a/libheif/color-conversion/hdr_sdr.cc
+++ b/libheif/color-conversion/hdr_sdr.cc
@@ -36,6 +36,16 @@ Op_to_hdr_planes::state_after_conversion(const ColorState& input_state,
return {};
}
+ // This operation increases the bit depth of an 8-bit input by replicating
+ // the input bit pattern: out = (in << (m-8)) | (in >> (16-m)). That identity
+ // only holds for target bit depths m in (8, 16]; a larger m would both make
+ // the right shift exponent negative (undefined behavior) and exceed the range
+ // of the uint16_t output plane. Only offer the conversion within that range.
+ if (target_state.bits_per_pixel <= 8 ||
+ target_state.bits_per_pixel > 16) {
+ return {};
+ }
+
std::vector<ColorStateWithCost> states;
ColorState output_state;
@@ -84,6 +94,17 @@ Op_to_hdr_planes::convert_colorspace(const std::shared_ptr<const HeifPixelImage>
int input_bits = input->get_bits_per_pixel(channel);
int output_bits = target_state.bits_per_pixel;
+ // Guard against unsupported bit-depth combinations. state_after_conversion()
+ // only offers this operation for 8-bit input and 8 < output <= 16, but a
+ // caller may invoke convert_colorspace() directly. Outside that range the
+ // bit-replication formula below would use a negative or oversized shift
+ // exponent (undefined behavior), so reject it instead.
+ if (input_bits != 8 || output_bits <= input_bits || output_bits > 2 * input_bits) {
+ return Error{heif_error_Unsupported_feature,
+ heif_suberror_Unsupported_color_conversion,
+ "Op_to_hdr_planes: unsupported bit depth conversion"};
+ }
+
int shift1 = output_bits - input_bits;
int shift2 = 2 * input_bits - output_bits;
diff --git a/tests/conversion.cc b/tests/conversion.cc
index fa970b66..a23e54dd 100644
--- a/tests/conversion.cc
+++ b/tests/conversion.cc
@@ -27,6 +27,7 @@
#include <iomanip>
#include "catch_amalgamated.hpp"
#include "color-conversion/colorconversion.h"
+#include "color-conversion/hdr_sdr.h"
#include "image/pixelimage.h"
#include <cmath>
@@ -896,6 +897,67 @@ TEST_CASE("Mismatched alpha bit depth - conversion correctness") {
}
+// Regression test for GHSA-8857-r8x5-7499. Op_to_hdr_planes widens an 8-bit input to a higher
+// bit depth with out = (in << (m-8)) | (in >> (16-m)). For m > 16 the right shift exponent
+// (16-m) becomes negative, which is undefined behavior (UBSan: "shift exponent is negative"),
+// and m > 16 also does not fit the uint16_t output plane. The operation must only offer/perform
+// the conversion for 8 < m <= 16.
+TEST_CASE("Op_to_hdr_planes rejects out-of-range output bit depth", "[heif_image]")
+{
+ heif_color_conversion_options options{};
+ std::unique_ptr<heif_color_conversion_options_ext, void(*)(heif_color_conversion_options_ext*)>
+ options_ext(heif_color_conversion_options_ext_alloc(), heif_color_conversion_options_ext_free);
+
+ ColorState input_state(heif_colorspace_YCbCr, heif_chroma_444, false, 8);
+ nclx_default_if_undefined(input_state);
+
+ const uint32_t width = 4;
+ const uint32_t height = 4;
+ auto img = std::make_shared<HeifPixelImage>();
+ img->create(width, height, heif_colorspace_YCbCr, heif_chroma_444);
+ img->fill_new_channel(heif_channel_Y, 0xAB, width, height, 8, nullptr);
+ img->fill_new_channel(heif_channel_Cb, 0xAB, width, height, 8, nullptr);
+ img->fill_new_channel(heif_channel_Cr, 0xAB, width, height, 8, nullptr);
+
+ Op_to_hdr_planes op;
+
+ SECTION("supported target bit depths widen correctly") {
+ for (int out_bits : {9, 10, 16}) {
+ ColorState target_state(heif_colorspace_YCbCr, heif_chroma_444, false, out_bits);
+ nclx_default_if_undefined(target_state);
+
+ auto states = op.state_after_conversion(input_state, target_state, options, *options_ext);
+ REQUIRE(states.size() == 1);
+
+ auto result = op.convert_colorspace(img, input_state, target_state, options, *options_ext,
+ heif_get_disabled_security_limits());
+ REQUIRE(result);
+ size_t stride;
+ const uint16_t* p = (const uint16_t*) (*result)->get_channel_memory(heif_channel_Y, &stride);
+ REQUIRE(p != nullptr);
+ const uint16_t expected = (uint16_t) ((0xAB << (out_bits - 8)) | (0xAB >> (16 - out_bits)));
+ CHECK(p[0] == expected);
+ }
+ }
+
+ SECTION("out-of-range target bit depths are rejected without UB") {
+ for (int out_bits : {17, 20, 24, 32}) {
+ ColorState target_state(heif_colorspace_YCbCr, heif_chroma_444, false, out_bits);
+ nclx_default_if_undefined(target_state);
+
+ // The pipeline must not select this operation for an unsupported target.
+ auto states = op.state_after_conversion(input_state, target_state, options, *options_ext);
+ CHECK(states.empty());
+
+ // A direct call must return an error instead of performing the negative shift.
+ auto result = op.convert_colorspace(img, input_state, target_state, options, *options_ext,
+ heif_get_disabled_security_limits());
+ CHECK_FALSE(result);
+ }
+ }
+}
+
+
// Regression test for GHSA-2c3g-p585-8rpq. Op_RGB24_32_to_YCbCr (like the other conversion
// operations) used to compute the row offset y * stride in 32-bit int, which overflows as soon
// as a plane exceeds 2 GB and made the conversion read from a wild address. All internal