Commit 08d6ed2100 for ffmpeg

commit 08d6ed2100b26c0161cab08570b3fbe70de0f765
Author: Michael Niedermayer <michael@niedermayer.cc>
Date:   Sun Nov 8 02:05:58 2026 +0100

    avcodec/encode: reject hardware frames smaller than the encoder

    Fixes: out of array read on the GPU
    Fixes: sXJ4jXv1fs6M

    Found during the review of https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25682

    Out of array read on the GPU Replicated through UnModified FFmpeg with
    the CUDA error reporting and the decoded output
    Out of array read on the GPU Replicated through API with the CUDA error
    reporting and the decoded output

diff --git a/libavcodec/encode.c b/libavcodec/encode.c
index c84cd87826..f0e68915a5 100644
--- a/libavcodec/encode.c
+++ b/libavcodec/encode.c
@@ -22,6 +22,7 @@
 #include "libavutil/channel_layout.h"
 #include "libavutil/emms.h"
 #include "libavutil/frame.h"
+#include "libavutil/hwcontext.h"
 #include "libavutil/internal.h"
 #include "libavutil/intreadwrite.h"
 #include "libavutil/mem.h"
@@ -522,9 +523,17 @@ static int encode_send_frame_internal(AVCodecContext *avctx, const AVFrame *src)
     } else if (!(avctx->codec->capabilities & AV_CODEC_CAP_PARAM_CHANGE)) {
         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(src->format);

-        if (!(desc && desc->flags & AV_PIX_FMT_FLAG_HWACCEL) &&
-            (src->format != avctx->pix_fmt ||
-             src->width != avctx->width || src->height != avctx->height)) {
+        if (desc && desc->flags & AV_PIX_FMT_FLAG_HWACCEL) {
+            if (src->hw_frames_ctx) {
+                const AVHWFramesContext *frames_ctx = (AVHWFramesContext *)src->hw_frames_ctx->data;
+
+                if (frames_ctx->width < avctx->width || frames_ctx->height < avctx->height) {
+                    av_log(avctx, AV_LOG_ERROR, "Hardware frames are smaller than the encoder\n");
+                    return AVERROR(EINVAL);
+                }
+            }
+        } else if (src->format != avctx->pix_fmt ||
+                   src->width != avctx->width || src->height != avctx->height) {
             av_log(avctx, AV_LOG_ERROR, "Frame size or pixel format does not match the encoder\n");
             return AVERROR(EINVAL);
         }