Commit f824f2e1cd for ffmpeg
commit f824f2e1cd46940754d5c5844113d61eb25d7e05
Author: Michael Niedermayer <michael@niedermayer.cc>
Date: Sat Nov 14 01:57:51 2026 +0100
avfilter/vf_hqdn3d: mask high bit depth samples to their depth
Fixes: out of array read
Fixes: 8Z3kXCidIqzE
Found-by: berrob14 (#24121)
Found independently during triage of the security report AISLE-2026-R0057
Out of array read Replicated through UnModified FFmpeg with ASAN
diff --git a/libavfilter/vf_hqdn3d.c b/libavfilter/vf_hqdn3d.c
index 3b46b6d4f8..e5c73d350d 100644
--- a/libavfilter/vf_hqdn3d.c
+++ b/libavfilter/vf_hqdn3d.c
@@ -42,7 +42,7 @@
#include "vf_hqdn3d.h"
#define LUT_BITS (depth==16 ? 8 : 4)
-#define LOAD(x) (((depth == 8 ? src[x] : AV_RN16A(src + (x) * 2)) << (16 - depth))\
+#define LOAD(x) (((depth == 8 ? src[x] : AV_RN16A(src + (x) * 2) & ((1 << depth) - 1)) << (16 - depth))\
+ (((1 << (16 - depth)) - 1) >> 1))
#define STORE(x,val) (depth == 8 ? dst[x] = (val) >> (16 - depth) : \
AV_WN16A(dst + (x) * 2, (val) >> (16 - depth)))
diff --git a/libavfilter/x86/vf_hqdn3d.asm b/libavfilter/x86/vf_hqdn3d.asm
index 1dfdc6c8fb..cbe21e057d 100644
--- a/libavfilter/x86/vf_hqdn3d.asm
+++ b/libavfilter/x86/vf_hqdn3d.asm
@@ -38,6 +38,9 @@ SECTION .text
movzx %1, word [srcq+(%2)*2]
%endif
%if %3 != 16
+%if %3 != 8
+ and %1, (1<<%3)-1
+%endif
shl %1, 16-%3
add %1, (1<<(15-%3))-1
%endif