Commit a7aeaecb77 for ffmpeg
commit a7aeaecb77fbe183b5ea1fa5fc8c30dbc46f88b7
Author: Michael Niedermayer <michael@niedermayer.cc>
Date: Mon Oct 26 16:02:26 2026 +0100
avfilter/vf_bm3d: compute MSE2SSE in double
Fixes: signed integer overflow
Found during triage/review of the security report fH770hlenwcI
Signed integer overflow Replicated through UnModified FFmpeg with UBSAN
diff --git a/libavfilter/vf_bm3d.c b/libavfilter/vf_bm3d.c
index 409c725956..f71845cd60 100644
--- a/libavfilter/vf_bm3d.c
+++ b/libavfilter/vf_bm3d.c
@@ -273,7 +273,7 @@ static void do_block_matching_multi(BM3DContext *s, const uint8_t *src, int src_
int r_y, int r_x, int plane, int jobnr)
{
SliceContext *sc = &s->slices[jobnr];
- double MSE2SSE = s->group_size * s->block_size * s->block_size * src_range * src_range / (double)(s->max * s->max);
+ double MSE2SSE = (double)s->group_size * s->block_size * s->block_size * src_range * src_range / ((double)s->max * s->max);
double distMul = 1. / MSE2SSE;
double th_sse = th_mse * MSE2SSE;
int index = sc->nb_match_blocks;