Commit 96b7f8281a for ffmpeg

commit 96b7f8281a5f971ef1b061024d50b971c85d4a95
Author: lusoris <lusoris@proton.me>
Date:   Thu Oct 22 20:41:33 2026 +0200

    avcodec/nvenc: drop hevc_nvenc SEI above a fixed size

    hevc_nvenc stops the encode with "Failed locking bitstream buffer: out
    of memory" once the SEI of a picture are large: nvEncLockBitstream()
    returns NV_ENC_ERR_OUT_OF_MEMORY. With driver 615.78.08 at 1920x1080 a
    single user data unregistered payload of up to 898 bytes passes on an
    IDR picture and up to 999 on a P picture, 672 and 747 when the payload
    is all zero bytes after the UUID. h264_nvenc carried 8192 bytes.

    Count the payload bytes of the A/53, timecode and user data unregistered
    SEI of an hevc_nvenc picture in order and drop a SEI that would take the
    total above 512 bytes, with a warning for the first one and verbose
    messages after that.

    Measured on an RTX 4090 with that driver, 1920x1080, five frames, one
    user data unregistered SEI on the IDR or on a P picture: payloads up to
    512 bytes give output byte-identical to the unpatched encoder, payloads
    of 513, 898, 999 and 1000 bytes are dropped with the warning and all
    five frames encode, and h264_nvenc output is unchanged.

    Signed-off-by: lusoris <lusoris@proton.me>

diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
index 6e7331405f..9a4e84f870 100644
--- a/libavcodec/nvenc.c
+++ b/libavcodec/nvenc.c
@@ -2726,6 +2726,41 @@ static int output_ready(AVCodecContext *avctx, int flush)
     return (nb_ready > 0) && (nb_ready + nb_pending >= ctx->async_depth);
 }

+/* hevc_nvenc fails nvEncLockBitstream() with NV_ENC_ERR_OUT_OF_MEMORY when
+ * the SEI of a picture are too large. With driver 615.78.08 at 1080p a single
+ * payload of up to 898 bytes passes on an IDR picture, 672 when it is all
+ * zero bytes; stay well below that. */
+#define NVENC_HEVC_SEI_MAX_PAYLOAD 512
+
+/* Drop the SEI that would take the total payload size of the picture above
+ * NVENC_HEVC_SEI_MAX_PAYLOAD and return the new count. */
+static int nvenc_hevc_sei_size_check(AVCodecContext *avctx, int count)
+{
+    NvencContext *ctx = avctx->priv_data;
+    size_t total = 0;
+    int kept = 0;
+
+    if (avctx->codec->id != AV_CODEC_ID_HEVC)
+        return count;
+
+    for (int i = 0; i < count; i++) {
+        NV_ENC_SEI_PAYLOAD *sei = &ctx->sei_data[i];
+
+        if (total + sei->payloadSize <= NVENC_HEVC_SEI_MAX_PAYLOAD) {
+            total += sei->payloadSize;
+            ctx->sei_data[kept++] = *sei;
+            continue;
+        }
+        av_log(avctx, ctx->sei_size_warned ? AV_LOG_VERBOSE : AV_LOG_WARNING,
+               "Not writing a %"PRIu32"-byte SEI of type %"PRIu32": hevc_nvenc "
+               "fails on more than about %d bytes of SEI per picture.\n",
+               sei->payloadSize, sei->payloadType, NVENC_HEVC_SEI_MAX_PAYLOAD);
+        ctx->sei_size_warned = 1;
+        av_freep(&sei->payload);
+    }
+    return kept;
+}
+
 static int prepare_sei_data_array(AVCodecContext *avctx, const AVFrame *frame)
 {
     NvencContext *ctx = avctx->priv_data;
@@ -2801,7 +2836,7 @@ static int prepare_sei_data_array(AVCodecContext *avctx, const AVFrame *frame)
     }

     if (!ctx->udu_sei)
-        return sei_count;
+        return nvenc_hevc_sei_size_check(avctx, sei_count);

     for (i = 0; i < frame->nb_side_data; i++) {
         AVFrameSideData *side_data = frame->side_data[i];
@@ -2831,7 +2866,7 @@ static int prepare_sei_data_array(AVCodecContext *avctx, const AVFrame *frame)
         }
     }

-    return sei_count;
+    return nvenc_hevc_sei_size_check(avctx, sei_count);

 error:
     for (i = 0; i < sei_count; i++)
diff --git a/libavcodec/nvenc.h b/libavcodec/nvenc.h
index acde2a42c1..7bb74a17a9 100644
--- a/libavcodec/nvenc.h
+++ b/libavcodec/nvenc.h
@@ -198,6 +198,7 @@ typedef struct NvencContext

     NV_ENC_SEI_PAYLOAD *sei_data;
     int sei_data_size;
+    int sei_size_warned; ///< a SEI was dropped by nvenc_hevc_sei_size_check()

     struct {
         void *ptr;