Commit 43d7598a for libheif

commit 43d7598acc47ca3ca72f95d0b77544b57c72aa6f
Author: Dirk Farin <dirk.farin@gmail.com>
Date:   Mon Oct 5 15:11:04 2026 +0200

    Decline filter-array images in Op_drop_alpha_plane and Op_adjust_alpha_bit_depth

    Both operations test the chroma format of the input state for the planar
    formats and not its colorspace. heif_chroma_monochrome is the same value
    as heif_chroma_planar, so a Bayer image (colorspace filter_array, chroma
    planar) with an alpha plane passed that test. Both operations declared
    the input state with another alpha plane, but convert_colorspace() only
    copies the Y, Cb, Cr, R, G and B planes. Op_drop_alpha_plane returned an
    image without any plane and Op_adjust_alpha_bit_depth returned the alpha
    plane alone, which contradicts the declared state.

    No conversion was affected. convert_colorspace() refuses a filter array
    with an alpha plane at its entry (468369af). Without that check, the
    pipeline for an RGB or YCbCr target starts with
    Op_bayer_bilinear_to_RGB24_32, which takes such an image directly. The two
    operations were only chosen for a conversion from a filter array to a
    filter array.

    Both operations now decline a filter-array input, like Op_to_hdr_planes,
    Op_to_sdr_planes and Op_flatten_alpha_plane already do.

    The test of the operation contract only used input states that pass
    check_plane_layout(), so it did not see this. It now also runs every
    operation against a filter array with an alpha plane, which
    HeifPixelImage accepts. Without this change it reports 42 contradictions
    in the two operations.

    The comment in check_plane_layout() said that nothing produces a filter
    array with an alpha plane. A filter-array item with an alpha auxiliary
    image is decoded to such an image.

diff --git a/libheif/color-conversion/alpha.cc b/libheif/color-conversion/alpha.cc
index b585a7e3..f49df004 100644
--- a/libheif/color-conversion/alpha.cc
+++ b/libheif/color-conversion/alpha.cc
@@ -76,6 +76,15 @@ Op_drop_alpha_plane::state_after_conversion(const ColorState& input_state,
     return {};
   }

+  // A Bayer image (colorspace filter_array with chroma planar, which has the same value as
+  // heif_chroma_monochrome) passes the chroma test above, but convert_colorspace() only copies
+  // the Y/Cb/Cr/R/G/B planes and would return an image without its filter-array plane. A filter
+  // array with an alpha plane is not a plane layout that the pipeline accepts
+  // (HeifPixelImage::check_plane_layout()), so decline it instead of declaring a state for it.
+  if (input_state.colorspace == heif_colorspace_filter_array) {
+    return {};
+  }
+
   if (options_ext.alpha_composition_mode != heif_alpha_composition_mode_none) {
     return {};
   }
@@ -453,6 +462,14 @@ Op_adjust_alpha_bit_depth::state_after_conversion(const ColorState& input_state,
     return {};
   }

+  // A Bayer image (colorspace filter_array with chroma planar, which has the same value as
+  // heif_chroma_monochrome) passes the chroma test above, but convert_colorspace() only copies
+  // the Y/Cb/Cr/R/G/B planes and would return the alpha plane alone. Decline it, like
+  // Op_drop_alpha_plane does.
+  if (input_state.colorspace == heif_colorspace_filter_array) {
+    return {};
+  }
+
   // Rewrites the alpha plane from its own bit depth to the colour bit depth, so both
   // ends have to be accessible as 8- or 16-bit samples.
   if (input_state.get_bytes_per_sample(heif_channel_Alpha) > 2 ||
diff --git a/libheif/image/pixelimage.cc b/libheif/image/pixelimage.cc
index 6084fbe7..11396f89 100644
--- a/libheif/image/pixelimage.cc
+++ b/libheif/image/pixelimage.cc
@@ -1035,10 +1035,12 @@ Error HeifPixelImage::check_plane_layout() const
         return layout_error("Chroma format is not valid for a filter-array image");
       }
       colour_planes = {heif_channel_filter_array};
-      // A filter array with an alpha plane is representable in 'unci', but nothing produces or
-      // consumes it yet: the uncompressed decoder only recognizes the filter-array component on
-      // its own, Op_bayer_bilinear_to_RGB24_32 carries no alpha, and Op_drop_alpha_plane does not
-      // copy a filter-array plane. Once those handle it, this may be allowed.
+      // A filter array with an alpha plane is representable in 'unci', but nothing converts it
+      // yet. The uncompressed decoder only recognizes the filter-array component on its own; a
+      // decoded image has both planes when a filter-array item has an alpha auxiliary image.
+      // Op_bayer_bilinear_to_RGB24_32 carries no alpha, and Op_drop_alpha_plane and
+      // Op_adjust_alpha_bit_depth decline filter arrays (they only copy the Y/Cb/Cr/R/G/B
+      // planes). Once those handle it, this may be allowed.
       separate_alpha_allowed = false;
       break;

diff --git a/tests/conversion_operator_contract.cc b/tests/conversion_operator_contract.cc
index bee7c455..4d8be692 100644
--- a/tests/conversion_operator_contract.cc
+++ b/tests/conversion_operator_contract.cc
@@ -228,6 +228,14 @@ std::shared_ptr<HeifPixelImage> make_bayer_image(const ColorState& state, uint32
   REQUIRE(mem != nullptr);
   fill_plane(mem, stride, w, h, bits, false);

+  if (state.bits_per_pixel_alpha) {
+    REQUIRE(!img->add_channel(heif_channel_Alpha, w, h, state.bits_per_pixel_alpha, heif_get_disabled_security_limits()));
+
+    mem = img->get_channel_memory(heif_channel_Alpha, &stride);
+    REQUIRE(mem != nullptr);
+    fill_plane(mem, stride, w, h, state.bits_per_pixel_alpha, std::endian::native == std::endian::big);
+  }
+
   return img;
 }

@@ -375,6 +383,32 @@ std::vector<ColorState> image_states()
 }


+// States that convert_colorspace() refuses at its entry (HeifPixelImage::check_plane_layout()),
+// but that an image can be built with, because HeifPixelImage accepts any set of planes: a
+// filter array with an alpha plane. What an operator declares for such an input has to hold as
+// well. Op_drop_alpha_plane and Op_adjust_alpha_bit_depth took the chroma format of a filter
+// array (planar, the same value as monochrome) for a monochrome image, declared the input
+// state with another alpha plane and returned an image without the filter-array plane.
+std::vector<ColorState> refused_input_states()
+{
+  std::vector<ColorState> states;
+
+  for (int bpp : {8, 10, 16}) {
+    for (int alpha_bpp : {8, 10, 16}) {
+      ColorState state;
+      state.colorspace = heif_colorspace_filter_array;
+      state.chroma = heif_chroma_planar;
+      state.bits_per_pixel_filter_array = bpp;
+      state.bits_per_pixel_alpha = alpha_bpp;
+      state.nclx = nclx_profile::defaults();
+      states.push_back(state);
+    }
+  }
+
+  return states;
+}
+
+
 struct Options
 {
   heif_color_conversion_options options;
@@ -405,9 +439,13 @@ std::vector<Options> option_sets()
 TEST_CASE("conversion operators return the image that they declared")
 {
   const std::vector<ColorState> targets = all_states();
-  const std::vector<ColorState> inputs = image_states();
   const heif_security_limits* limits = heif_get_disabled_security_limits();

+  std::vector<ColorState> inputs = image_states();
+  for (const ColorState& state : refused_input_states()) {
+    inputs.push_back(state);
+  }
+
   size_t num_operators_used = 0;
   size_t num_conversions = 0;
   size_t num_contradictions = 0;