Commit c4769e5a for libheif
commit c4769e5a395675b7710b0189f30042d13e3a1e4d
Author: Dirk Farin <dirk.farin@gmail.com>
Date: Sun Sep 20 14:18:02 2026 +0200
Fix a null-pointer write when compositing alpha into a YCbCr target
Op_flatten_alpha_plane allocated its composited R/G/B planes with
target_state.get_bits_per_pixel(channel). Since the per-plane ColorState
(e379fd51) those fields are 0 for a YCbCr or monochrome target, because
the operation keeps the source colorspace. add_channel() refused the zero
depth, the error was dropped, and the composite loop wrote through a null
plane pointer. Every compositing decode into a YCbCr target crashed, for
example heif-dec writing a JPEG from any HEIC or AVIF with an alpha
channel. RGB targets were unaffected because the planner converts to RGB
before it flattens.
Allocate the planes with the depth of the RGB-converted input and
propagate the add_channel() error.
Regression tests: conversion.cc composites synthesized 8-bit and 10-bit
YCbCr images into YCbCr and RGB targets and pins that a monochrome target
declines cleanly (compositing a monochrome image has never been supported
because there is no RGB to monochrome operator); alpha_composite_decode
decodes the shipped HEVC and AV1 alpha samples the way heif-dec does.
diff --git a/libheif/color-conversion/alpha.cc b/libheif/color-conversion/alpha.cc
index 9055b70f..966b37c6 100644
--- a/libheif/color-conversion/alpha.cc
+++ b/libheif/color-conversion/alpha.cc
@@ -219,7 +219,14 @@ Op_flatten_alpha_plane<Pixel>::convert_colorspace(const std::shared_ptr<const He
for (heif_channel channel : {heif_channel_R,
heif_channel_G,
heif_channel_B}) {
- outimg->add_channel(channel, width, height, target_state.get_bits_per_pixel(channel), limits);
+ // 'input' was converted to planar RGB above, so its planes carry the depth we composite
+ // at. Do not take the depth from target_state: this operation keeps the source
+ // colorspace, so for a YCbCr or monochrome target the R/G/B fields there are 0 (plane
+ // absent). add_channel() refuses a zero depth and the loops below would then write
+ // through a null plane pointer.
+ if (Error err = outimg->add_channel(channel, width, height, input->get_bits_per_pixel(channel), limits)) {
+ return err;
+ }
const Pixel* p_alpha;
size_t stride_alpha;
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 67160b76..32eaed50 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -100,6 +100,7 @@ add_libheif_test(item_writing)
add_libheif_test(overlay_amplification)
add_libheif_test(error_item_decode)
add_libheif_test(alpha_cycle_deadlock)
+add_libheif_test(alpha_composite_decode)
add_libheif_test(parallel_grid_deadlock)
# The deadlock regression tests decode on a worker thread guarded by a timeout.
find_package(Threads REQUIRED)
diff --git a/tests/alpha_composite_decode.cc b/tests/alpha_composite_decode.cc
new file mode 100644
index 00000000..f14d0668
--- /dev/null
+++ b/tests/alpha_composite_decode.cc
@@ -0,0 +1,127 @@
+/*
+ libheif unit tests
+
+ MIT License
+
+ Copyright (c) 2026 Dirk Farin <dirk.farin@gmail.com>
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in all
+ copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ SOFTWARE.
+*/
+
+// Regression test: decoding an image that has an alpha channel into a YCbCr target
+// while compositing the alpha onto a background (what heif-dec does when it writes
+// JPEG) crashed with a null-pointer write in Op_flatten_alpha_plane. The operator
+// allocated its composited R/G/B planes with the depth recorded for those channels
+// in the target ColorState, which is 0 for a YCbCr target. Decoding to an RGB
+// target was unaffected because the planner converts to RGB before it flattens.
+//
+// The companion test in conversion.cc drives the operator directly on synthesized
+// images. This one goes through the public decoding API on the shipped alpha samples
+// (HEVC and AV1), so it also covers the planner route a real decode takes.
+
+#include "catch_amalgamated.hpp"
+#include "libheif/heif.h"
+#include "test_utils.h"
+
+namespace {
+
+struct Sample
+{
+ const char* filename;
+ heif_compression_format codec;
+};
+
+const Sample samples[] = {
+ {"simple_osm_tile_alpha.avif", heif_compression_AV1},
+ {"with-alpha-512x512.heic", heif_compression_HEVC},
+};
+
+void decode_composited(const char* filename,
+ heif_colorspace colorspace, heif_chroma chroma,
+ heif_alpha_composition_mode mode)
+{
+ INFO(filename << " -> colorspace " << colorspace << ", chroma " << chroma << ", mode " << mode);
+
+ heif_context* ctx = get_context_for_test_file(filename);
+ heif_image_handle* handle = get_primary_image_handle(ctx);
+ REQUIRE(heif_image_handle_has_alpha_channel(handle));
+
+ heif_decoding_options* options = heif_decoding_options_alloc();
+ REQUIRE(options != nullptr);
+
+ heif_color_conversion_options_ext* ext = heif_color_conversion_options_ext_alloc();
+ REQUIRE(ext != nullptr);
+ ext->alpha_composition_mode = mode;
+ ext->background_red = 0xFFFF;
+ ext->background_green = 0xFFFF;
+ ext->background_blue = 0xFFFF;
+ ext->secondary_background_red = 0x8000;
+ ext->secondary_background_green = 0x8000;
+ ext->secondary_background_blue = 0x8000;
+ ext->checkerboard_square_size = 16;
+ options->color_conversion_options_ext = ext;
+
+ heif_image* img = nullptr;
+ heif_error err = heif_decode_image(handle, &img, colorspace, chroma, options);
+ INFO("decode error (" << err.code << "/" << err.subcode << "): " << err.message);
+ REQUIRE(err.code == heif_error_Ok);
+ REQUIRE(img != nullptr);
+
+ CHECK(heif_image_get_colorspace(img) == colorspace);
+ CHECK(heif_image_get_chroma_format(img) == chroma);
+
+ // The alpha has been composited onto the colour planes, so none is left.
+ CHECK(!heif_image_has_channel(img, heif_channel_Alpha));
+
+ heif_channel size_channel = (colorspace == heif_colorspace_YCbCr) ? heif_channel_Y : heif_channel_interleaved;
+ CHECK(heif_image_get_width(img, size_channel) == heif_image_handle_get_width(handle));
+ CHECK(heif_image_get_height(img, size_channel) == heif_image_handle_get_height(handle));
+
+ heif_image_release(img);
+ heif_color_conversion_options_ext_free(ext);
+ heif_decoding_options_free(options);
+ heif_image_handle_release(handle);
+ heif_context_free(ctx);
+}
+
+} // namespace
+
+TEST_CASE("decode with alpha compositing into a YCbCr target")
+{
+ int samples_tested = 0;
+
+ for (const Sample& sample : samples) {
+ if (!heif_have_decoder_for_format(sample.codec)) {
+ continue;
+ }
+ samples_tested++;
+
+ // heif-dec's default when the output format cannot store alpha.
+ decode_composited(sample.filename, heif_colorspace_YCbCr, heif_chroma_420, heif_alpha_composition_mode_checkerboard);
+ decode_composited(sample.filename, heif_colorspace_YCbCr, heif_chroma_420, heif_alpha_composition_mode_solid_color);
+ decode_composited(sample.filename, heif_colorspace_YCbCr, heif_chroma_444, heif_alpha_composition_mode_solid_color);
+
+ // An RGB target takes the other planner route (convert first, flatten on RGB).
+ decode_composited(sample.filename, heif_colorspace_RGB, heif_chroma_interleaved_RGB, heif_alpha_composition_mode_checkerboard);
+ }
+
+ if (samples_tested == 0) {
+ SKIP("neither an AV1 nor a HEVC decoder is available");
+ }
+}
diff --git a/tests/conversion.cc b/tests/conversion.cc
index 7151f399..e2d15417 100644
--- a/tests/conversion.cc
+++ b/tests/conversion.cc
@@ -1135,3 +1135,193 @@ TEST_CASE("RGB24 to YCbCr conversion with planes larger than 2 GB", "[.large-mem
REQUIRE(big_y[static_cast<size_t>(h - 1) * big_stride + (w - 1)] == small_y[small_stride]);
REQUIRE(small_y[0] != small_y[small_stride]);
}
+
+
+// Regression test for a null-pointer write introduced together with the per-plane
+// ColorState (bits_per_pixel_R/G/B/Y/Cb/Cr/alpha).
+//
+// Op_flatten_alpha_plane composites the alpha plane onto the colour planes when the output
+// cannot carry alpha. It converts its input to planar RGB, composites, and converts the
+// result back to the input's colorspace. The composited R/G/B planes were allocated with
+// target_state.get_bits_per_pixel(channel). The operation keeps the source colorspace, so
+// for a YCbCr (or monochrome) source the R/G/B fields of target_state are 0 (plane absent):
+// add_channel() refused the zero depth, the returned error was dropped, and the composite
+// loop wrote through a null plane pointer. Every compositing decode whose target is YCbCr
+// crashed (heif-dec writing JPEG from any alpha HEIC or AVIF), while an RGB target was
+// unaffected because the planner converts to RGB before it flattens. The planes are now
+// allocated with the depth of the RGB-converted input and the add_channel() error is
+// propagated.
+
+static std::shared_ptr<HeifPixelImage> make_ycbcr_alpha_image(heif_chroma chroma,
+ uint32_t width, uint32_t height,
+ int bpp, uint16_t luma, uint16_t alpha,
+ const nclx_profile& nclx)
+{
+ auto img = std::make_shared<HeifPixelImage>();
+ img->create(width, height, heif_colorspace_YCbCr, chroma);
+
+ uint32_t chroma_width = (chroma == heif_chroma_444) ? width : (width + 1) / 2;
+ uint32_t chroma_height = (chroma == heif_chroma_420) ? (height + 1) / 2 : height;
+ uint16_t neutral = static_cast<uint16_t>(1 << (bpp - 1));
+
+ img->fill_new_channel(heif_channel_Y, luma, width, height, bpp, nullptr);
+ img->fill_new_channel(heif_channel_Cb, neutral, chroma_width, chroma_height, bpp, nullptr);
+ img->fill_new_channel(heif_channel_Cr, neutral, chroma_width, chroma_height, bpp, nullptr);
+ img->fill_new_channel(heif_channel_Alpha, alpha, width, height, bpp, nullptr);
+ img->set_color_profile_nclx(nclx);
+ return img;
+}
+
+// The operator computes (p * a + bkg * (alpha_max - a)) >> bpp_alpha. With neutral chroma
+// and a full-range BT.601 matrix, the RGB gray level equals the luma, so the composited
+// luma is predictable up to the rounding of the two matrix conversions.
+static int expected_composited_luma(int luma, int alpha, int background, int bpp)
+{
+ int alpha_max = (1 << bpp) - 1;
+ return (luma * alpha + background * (alpha_max - alpha)) >> bpp;
+}
+
+TEST_CASE("Alpha compositing into a YCbCr or monochrome target", "[heif_image]")
+{
+ const uint32_t width = 16;
+ const uint32_t height = 8;
+
+ heif_color_conversion_options options{};
+
+ heif_color_conversion_options_ext options_ext{};
+ options_ext.version = 1;
+ options_ext.alpha_composition_mode = heif_alpha_composition_mode_solid_color;
+ options_ext.background_red = 0xFFFF;
+ options_ext.background_green = 0xFFFF;
+ options_ext.background_blue = 0xFFFF;
+
+ nclx_profile nclx = nclx_profile::defaults();
+ nclx.set_matrix_coefficients(heif_matrix_coefficients_ITU_R_BT_601_6);
+ nclx.set_full_range_flag(true);
+
+ SECTION("8-bit YCbCr 4:2:0 -> YCbCr 4:2:0 with a solid background (heif-dec JPEG output)") {
+ auto img = make_ycbcr_alpha_image(heif_chroma_420, width, height, 8, 100, 128, nclx);
+
+ auto result = convert_colorspace(img, heif_colorspace_YCbCr, heif_chroma_420,
+ nclx, 0, options, &options_ext,
+ heif_get_disabled_security_limits());
+ REQUIRE(result);
+ auto out = *result;
+
+ CHECK(out->get_colorspace() == heif_colorspace_YCbCr);
+ CHECK(out->get_chroma_format() == heif_chroma_420);
+ CHECK(!out->has_channel(heif_channel_Alpha));
+ REQUIRE(out->has_channel(heif_channel_Y));
+ CHECK(out->get_bits_per_pixel(heif_channel_Y) == 8);
+
+ size_t stride;
+ const uint8_t* p_y = out->get_channel_memory(heif_channel_Y, &stride);
+ REQUIRE(p_y != nullptr);
+
+ int expected = expected_composited_luma(100, 128, 255, 8);
+ CHECK(std::abs(p_y[0] - expected) <= 2);
+ CHECK(std::abs(p_y[(height - 1) * stride + (width - 1)] - expected) <= 2);
+ }
+
+ SECTION("10-bit YCbCr 4:2:0 -> YCbCr 4:2:0 with a solid background (16-bit operator)") {
+ auto img = make_ycbcr_alpha_image(heif_chroma_420, width, height, 10, 400, 512, nclx);
+
+ auto result = convert_colorspace(img, heif_colorspace_YCbCr, heif_chroma_420,
+ nclx, 0, options, &options_ext,
+ heif_get_disabled_security_limits());
+ REQUIRE(result);
+ auto out = *result;
+
+ CHECK(out->get_chroma_format() == heif_chroma_420);
+ CHECK(!out->has_channel(heif_channel_Alpha));
+ REQUIRE(out->has_channel(heif_channel_Y));
+ CHECK(out->get_bits_per_pixel(heif_channel_Y) == 10);
+
+ size_t stride;
+ const uint8_t* p_y = out->get_channel_memory(heif_channel_Y, &stride);
+ REQUIRE(p_y != nullptr);
+ const uint16_t* y16 = reinterpret_cast<const uint16_t*>(p_y);
+
+ int expected = expected_composited_luma(400, 512, 1023, 10);
+ CHECK(std::abs(y16[0] - expected) <= 3);
+ CHECK(std::abs(y16[(height - 1) * (stride / 2) + (width - 1)] - expected) <= 3);
+ }
+
+ SECTION("8-bit YCbCr 4:4:4 -> YCbCr 4:4:4 with a checkerboard") {
+ heif_color_conversion_options_ext checker = options_ext;
+ checker.alpha_composition_mode = heif_alpha_composition_mode_checkerboard;
+ checker.secondary_background_red = 0;
+ checker.secondary_background_green = 0;
+ checker.secondary_background_blue = 0;
+ checker.checkerboard_square_size = 8;
+
+ auto img = make_ycbcr_alpha_image(heif_chroma_444, width, height, 8, 100, 128, nclx);
+
+ auto result = convert_colorspace(img, heif_colorspace_YCbCr, heif_chroma_444,
+ nclx, 0, options, &checker,
+ heif_get_disabled_security_limits());
+ REQUIRE(result);
+ auto out = *result;
+
+ CHECK(out->get_chroma_format() == heif_chroma_444);
+ CHECK(!out->has_channel(heif_channel_Alpha));
+ REQUIRE(out->has_channel(heif_channel_Y));
+
+ size_t stride;
+ const uint8_t* p_y = out->get_channel_memory(heif_channel_Y, &stride);
+ REQUIRE(p_y != nullptr);
+
+ // parity = (x/8 + y/8) % 2 selects the secondary (black) square at (0,0) and the
+ // primary (white) square at (8,0).
+ int expected_black = expected_composited_luma(100, 128, 0, 8);
+ int expected_white = expected_composited_luma(100, 128, 255, 8);
+ CHECK(std::abs(p_y[0] - expected_black) <= 2);
+ CHECK(std::abs(p_y[8] - expected_white) <= 2);
+ }
+
+ SECTION("8-bit YCbCr 4:2:0 -> interleaved RGB (RGB target, was not affected)") {
+ auto img = make_ycbcr_alpha_image(heif_chroma_420, width, height, 8, 100, 128, nclx);
+
+ auto result = convert_colorspace(img, heif_colorspace_RGB, heif_chroma_interleaved_RGB,
+ nclx, 0, options, &options_ext,
+ heif_get_disabled_security_limits());
+ REQUIRE(result);
+ auto out = *result;
+
+ CHECK(out->get_chroma_format() == heif_chroma_interleaved_RGB);
+
+ size_t stride;
+ const uint8_t* p = out->get_channel_memory(heif_channel_interleaved, &stride);
+ REQUIRE(p != nullptr);
+
+ int expected = expected_composited_luma(100, 128, 255, 8);
+ CHECK(std::abs(p[0] - expected) <= 2);
+ CHECK(std::abs(p[1] - expected) <= 2);
+ CHECK(std::abs(p[2] - expected) <= 2);
+ }
+
+ SECTION("8-bit monochrome -> monochrome with a solid background") {
+ // The crash also covered a monochrome target (bits_per_pixel_R/G/B are 0 there too).
+ // Compositing a monochrome image is not actually supported: the operator converts
+ // the input to RGB, composites, and has no operator to convert back to monochrome,
+ // so the request fails with Unsupported_color_conversion (unchanged from v1.23.4).
+ // Pin that it declines cleanly instead of crashing; if monochrome compositing is
+ // implemented, replace the error check by a check of the composited luma.
+ auto img = std::make_shared<HeifPixelImage>();
+ img->create(width, height, heif_colorspace_monochrome, heif_chroma_monochrome);
+ img->fill_new_channel(heif_channel_Y, 100, width, height, 8, nullptr);
+ img->fill_new_channel(heif_channel_Alpha, 128, width, height, 8, nullptr);
+ img->set_color_profile_nclx(nclx);
+
+ auto result = convert_colorspace(img, heif_colorspace_monochrome, heif_chroma_monochrome,
+ nclx, 0, options, &options_ext,
+ heif_get_disabled_security_limits());
+ if (result) {
+ CHECK(!(*result)->has_channel(heif_channel_Alpha));
+ }
+ else {
+ CHECK(result.error().error_code == heif_error_Unsupported_feature);
+ CHECK(result.error().sub_error_code == heif_suberror_Unsupported_color_conversion);
+ }
+ }
+}