Commit c4341013 for libheif

commit c43410130223809ea6d0754d71781a2f656cda84
Author: Dirk Farin <dirk.farin@gmail.com>
Date:   Sun Sep 6 15:20:56 2026 +0200

    x264: encode 10 bit input instead of mangling it

    The plugin took the bit depth from the image and then applied the profiles
    "main10-intra" and "main12-intra" for 10 and 12 bits. Those are x265 profile
    names; x264_profile_names[] holds "baseline", "main", "high", "high10",
    "high422" and "high444", so x264_param_apply_profile() returned -1 and did
    nothing. Nothing else told libx264 about the depth either: param.i_bitdepth was
    left at its default and X264_CSP_HIGH_DEPTH was never added to param.i_csp, so
    libx264 read the two byte per sample planes one byte at a time and encoded the
    result as an 8 bit image. A flat 10 bit mid grey came back out of the round trip
    as a stripe pattern.

    Set param.i_bitdepth, add X264_CSP_HIGH_DEPTH for input deeper than 8 bits, and
    apply "high10" rather than a name x264 does not know. 12 bits is refused: x264
    has 8 and 10 bit modes and nothing else. param.i_bitdepth arrived in x264 build
    153, and an older build encodes at whatever depth it was compiled for, so those
    stay at 8 bits.

    A 10 bit encode now produces a High 10 stream with bit_depth_luma 10 in the
    'avcC' and round trips to the input value, in 4:2:0, 4:2:2, 4:4:4 and greyscale.

diff --git a/libheif/plugins/encoder_x264.cc b/libheif/plugins/encoder_x264.cc
index d937e4d4..fea7696b 100644
--- a/libheif/plugins/encoder_x264.cc
+++ b/libheif/plugins/encoder_x264.cc
@@ -222,6 +222,8 @@ parameter encoder_struct_x264::get_param(const std::string& name) const
 }


+static const char* kError_unsupported_bit_depth = "Bit depth not supported by x264";
+
 static const char* kParam_preset = "preset";
 static const char* kParam_tune = "tune";
 static const char* kParam_TU_intra_depth = "tu-intra-depth";
@@ -733,14 +735,34 @@ static heif_error x264_start_sequence_encoding_intern(void* encoder_raw, const h

   x264_param_default_preset(&param, encoder->preset.c_str(), encoder->tune.c_str());

-  if (bit_depth == 8) {
-    x264_param_apply_profile(&param, "main");
+  // x264 encodes 8 and 10 bits, and nothing else. Which of the two a build offers
+  // is decided by param.i_bitdepth, which exists since x264 build 153; older
+  // builds are fixed to whatever they were compiled for and have no 10 bit mode
+  // we could ask for. x264_encoder_open() below fails if the depth is not there.
+  if (bit_depth != 8 && bit_depth != 10) {
+    return heif_error{
+      heif_error_Encoder_plugin_error,
+      heif_suberror_Unsupported_bit_depth,
+      kError_unsupported_bit_depth
+    };
   }
-  else if (bit_depth == 10) x264_param_apply_profile(&param, "main10-intra");
-  else if (bit_depth == 12) x264_param_apply_profile(&param, "main12-intra");
-  else {
-    return heif_error_unsupported_parameter;
+
+#if X264_BUILD >= 153
+  param.i_bitdepth = bit_depth;
+#else
+  if (bit_depth != 8) {
+    return heif_error{
+      heif_error_Encoder_plugin_error,
+      heif_suberror_Unsupported_bit_depth,
+      kError_unsupported_bit_depth
+    };
   }
+#endif
+
+  // Applied before param.i_csp is set below, so it constrains the coding tools
+  // and the bit depth only. x264 still picks the profile that the final chroma
+  // format needs.
+  x264_param_apply_profile(&param, bit_depth > 8 ? "high10" : "main");


   param.i_fps_num = framerate_num;
@@ -788,6 +810,13 @@ static heif_error x264_start_sequence_encoding_intern(void* encoder_raw, const h
     param.i_csp = X264_CSP_I444;
   }

+  // Tells x264 that the input planes hold 16 bit samples. HeifPixelImage stores
+  // anything above 8 bits at two bytes per sample, so this has to follow the bit
+  // depth of the image, not the depth we encode at.
+  if (bit_depth > 8) {
+    param.i_csp |= X264_CSP_HIGH_DEPTH;
+  }
+
   if (chroma != heif_chroma_monochrome) {
     int w = heif_image_get_width(image, heif_channel_Y);
     int h = heif_image_get_height(image, heif_channel_Y);
@@ -960,9 +989,10 @@ static heif_error x264_encode_sequence_frame(void* encoder_raw, const heif_image
                                              uintptr_t frame_nr)
 {
   // H.264 can signal different luma and chroma bit depths, but x264 has a
-  // single bit depth and cannot produce such a stream.
+  // single bit depth and cannot produce such a stream. x264 has 8 and 10 bit
+  // modes and no others.
   heif_error input_error = check_encoder_input_image(image, /*supports_monochrome=*/true,
-                                                    {8, 10, 12});
+                                                    {8, 10});
   if (input_error.code != heif_error_Ok) {
     return input_error;
   }
@@ -978,6 +1008,13 @@ static heif_error x264_encode_sequence_frame(void* encoder_raw, const heif_image
     };
   }

+  // pic.img.i_csp below carries the X264_CSP_HIGH_DEPTH flag of the first frame
+  // of the sequence, while the plane pointers are this frame's.
+  input_error = check_sequence_frame_bit_depth(image, encoder->bit_depth);
+  if (input_error.code != heif_error_Ok) {
+    return input_error;
+  }
+
   heif_error err;

   // Note: it is ok to cast away the const, as the image content is not changed.