Commit 744588782d for ffmpeg

commit 744588782d47071a64b963edfbeedb052ee9440c
Author: Michael Niedermayer <michael@niedermayer.cc>
Date:   Mon Oct 12 04:06:17 2026 +0200

    swscale: reject planes whose size does not fit in int

    Fixes: signed integer overflow
    Fixes: out of array access
    Fixes: pdA1vTaOO5Be
    Found during the review of https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24851
    Signed integer overflow Replicated through API with UBSAN
    Out of array access Replicated through API with ASAN

diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 80b30bcf69..8d92c57c64 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -726,7 +726,7 @@ static void reset_ptr(const uint8_t *src[], enum AVPixelFormat format)
 }

 static int check_image_pointers(const uint8_t * const data[4], enum AVPixelFormat pix_fmt,
-                                const int linesizes[4])
+                                const int linesizes[4], int h)
 {
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
     int i;
@@ -735,7 +735,7 @@ static int check_image_pointers(const uint8_t * const data[4], enum AVPixelForma

     for (i = 0; i < 4; i++) {
         int plane = desc->comp[i].plane;
-        if (!data[plane] || !linesizes[plane])
+        if (!data[plane] || !linesizes[plane] || FFABS((int64_t)linesizes[plane]) * h > INT_MAX)
             return 0;
     }

@@ -1060,12 +1060,12 @@ static int scale_internal(SwsContext *sws,
         return AVERROR(EINVAL);
     }

-    if (!check_image_pointers(srcSlice, sws->src_format, srcStride)) {
-        av_log(c, AV_LOG_ERROR, "bad src image pointers\n");
+    if (!check_image_pointers(srcSlice, sws->src_format, srcStride, sws->src_h)) {
+        av_log(c, AV_LOG_ERROR, "bad src image pointers or strides\n");
         return AVERROR(EINVAL);
     }
-    if (!check_image_pointers((const uint8_t* const*)dstSlice, sws->dst_format, dstStride)) {
-        av_log(c, AV_LOG_ERROR, "bad dst image pointers\n");
+    if (!check_image_pointers((const uint8_t* const*)dstSlice, sws->dst_format, dstStride, sws->dst_h)) {
+        av_log(c, AV_LOG_ERROR, "bad dst image pointers or strides\n");
         return AVERROR(EINVAL);
     }

@@ -1108,7 +1108,7 @@ static int scale_internal(SwsContext *sws,
         int x,y;

         av_fast_malloc(&c->rgb0_scratch, &c->rgb0_scratch_allocated,
-                       FFABS(srcStride[0]) * srcSliceH + 32);
+                       FFABS((int64_t)srcStride[0]) * srcSliceH + 32);
         if (!c->rgb0_scratch)
             return AVERROR(ENOMEM);

@@ -1127,7 +1127,7 @@ static int scale_internal(SwsContext *sws,
         uint8_t *base;

         av_fast_malloc(&c->xyz_scratch, &c->xyz_scratch_allocated,
-                       FFABS(srcStride[0]) * srcSliceH + 32);
+                       FFABS((int64_t)srcStride[0]) * srcSliceH + 32);
         if (!c->xyz_scratch)
             return AVERROR(ENOMEM);

@@ -1402,6 +1402,14 @@ int sws_receive_slice(SwsContext *sws, unsigned int slice_start,
                           dst, c->frame_dst->linesize, slice_start, slice_height);
 }

+static int frame_strides_fit(const AVFrame *frame)
+{
+    for (int i = 0; i < FF_ARRAY_ELEMS(frame->linesize); i++)
+        if (FFABS((int64_t)frame->linesize[i]) * frame->height > INT_MAX)
+            return 0;
+    return 1;
+}
+
 int sws_scale_frame(SwsContext *sws, AVFrame *dst, const AVFrame *src)
 {
     int ret, allocated = 0;
@@ -1432,6 +1440,11 @@ int sws_scale_frame(SwsContext *sws, AVFrame *dst, const AVFrame *src)
     if (!src->data[0])
         return 0;

+    if (!frame_strides_fit(src) || (dst->data[0] && !frame_strides_fit(dst))) {
+        av_log(sws, AV_LOG_ERROR, "Frame planes larger than INT_MAX bytes are not supported\n");
+        return AVERROR(EINVAL);
+    }
+
     const SwsGraph *top = c->graph[FIELD_TOP];
     const SwsGraph *bot = c->graph[FIELD_BOTTOM];
     if (dst->data[0]) /* user-provided buffers */