Commit b47e96b4eb for ffmpeg

commit b47e96b4eb50fef52cf1a9c7b0100a34cc05cce0
Author: Michael Niedermayer <michael@niedermayer.cc>
Date:   Thu Oct 29 03:09:46 2026 +0100

    avcodec/pcxenc: do not read the scanline padding from the frame

    Encode the pixels of the plane and a zero padding byte instead.

    Fixes: out of array read
    Fixes: oHpVCeufOIJY
    Fixes: AISLE-2026-0150-00405
    Out of array read Replicated through API with ASAN

    Found-by: Joshua Rogers (AISLE)

diff --git a/libavcodec/pcxenc.c b/libavcodec/pcxenc.c
index f450d7bca1..3525d33564 100644
--- a/libavcodec/pcxenc.c
+++ b/libavcodec/pcxenc.c
@@ -34,12 +34,21 @@

 static const uint32_t monoblack_pal[16] = { 0x000000, 0xFFFFFF };

+static uint8_t *put_run(uint8_t *dst, uint8_t value, int count)
+{
+    if (count != 1 || value >= 0xC0)
+        *dst++ = 0xC0 | count;
+    *dst++ = value;
+    return dst;
+}
+
 /**
  * PCX run-length encoder
  * @param dst output buffer
  * @param dst_size size of output buffer
  * @param src input buffer
- * @param src_plane_size size of one plane of input buffer in bytes
+ * @param src_plane_size size of one plane of input buffer in bytes, an odd
+ *                       size is padded with a zero byte
  * @param nplanes number of planes in input buffer
  * @return number of bytes written to dst or -1 on error
  * @bug will not work for nplanes != 1 && bpp != 8
@@ -49,36 +58,35 @@ static int pcx_rle_encode(      uint8_t *dst, int dst_size,
 {
     int p;
     const uint8_t *dst_start = dst;
+    int pad = src_plane_size & 1;

     // check worst-case upper bound on dst_size
-    if (dst_size < 2LL * src_plane_size * nplanes || src_plane_size <= 0)
+    if (dst_size < 2LL * (src_plane_size + pad) * nplanes || src_plane_size <= 0)
         return AVERROR(EINVAL);

     for (p = 0; p < nplanes; p++) {
-        int count = 1;
+        int count = 1, i, zero_pad = pad;
         const uint8_t *src_plane = src + p;
-        const uint8_t *src_plane_end = src_plane + src_plane_size * nplanes;
         uint8_t prev = *src_plane;
-        src_plane += nplanes;

-        for (; ; src_plane += nplanes) {
-            if (src_plane < src_plane_end && *src_plane == prev && count < 0x3F) {
-                // current byte is same as prev
+        for (i = 1; i < src_plane_size; i++) {
+            uint8_t cur = src_plane[i * nplanes];
+
+            if (cur == prev && count < 0x3F) {
                 ++count;
-            } else {
-                // output prev * count
-                if (count != 1 || prev >= 0xC0)
-                    *dst++ = 0xC0 | count;
-                *dst++ = prev;
-
-                if (src_plane == src_plane_end)
-                    break;
-
-                // start new run
-                count = 1;
-                prev = *src_plane;
+                continue;
             }
+            dst = put_run(dst, prev, count);
+            count = 1;
+            prev = cur;
+        }
+        if (zero_pad && !prev && count < 0x3F) {
+            count++;
+            zero_pad = 0;
         }
+        dst = put_run(dst, prev, count);
+        if (zero_pad)
+            *dst++ = 0;
     }

     return dst - dst_start;
@@ -90,7 +98,7 @@ static int pcx_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     const uint8_t *buf_end;
     uint8_t *buf;

-    int bpp, nplanes, i, y, line_bytes, written, ret, max_pkt_size, sw, sh;
+    int bpp, nplanes, i, y, line_bytes, src_plane_size, written, ret, max_pkt_size, sw, sh;
     const uint32_t *pal = NULL;
     uint32_t palette256[256];
     const uint8_t *src;
@@ -130,8 +138,8 @@ static int pcx_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
         return AVERROR(EINVAL);
     }

-    line_bytes = (avctx->width * bpp + 7) >> 3;
-    line_bytes = (line_bytes + 1) & ~1;
+    src_plane_size = (avctx->width * bpp + 7) >> 3;
+    line_bytes = (src_plane_size + 1) & ~1;

     max_pkt_size = 128 + avctx->height * 2 * line_bytes * nplanes + (pal ? 256*3 + 1 : 0);
     if ((ret = ff_alloc_packet(avctx, pkt, max_pkt_size)) < 0)
@@ -167,7 +175,7 @@ static int pcx_encode_frame(AVCodecContext *avctx, AVPacket *pkt,

     for (y = 0; y < avctx->height; y++) {
         if ((written = pcx_rle_encode(buf, buf_end - buf,
-                                      src, line_bytes, nplanes)) < 0) {
+                                      src, src_plane_size, nplanes)) < 0) {
             av_log(avctx, AV_LOG_ERROR, "buffer too small\n");
             return AVERROR_BUG;
         }