From dd6040675ec18d19429f882caea6bb306ed6677a Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Wed, 18 May 2022 02:10:52 +0200 Subject: avfilter/vf_signature: Fix integer overflow in filter_frame() Fixes: CID1403233 The second of the 2 changes may be unneeded but will help coverity Signed-off-by: Michael Niedermayer --- libavfilter/vf_signature.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libavfilter') diff --git a/libavfilter/vf_signature.c b/libavfilter/vf_signature.c index 5cff149756..7d434328b7 100644 --- a/libavfilter/vf_signature.c +++ b/libavfilter/vf_signature.c @@ -219,7 +219,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *picref) dw1 = inlink->w / 32; if (inlink->w % 32) dw2 = dw1 + 1; - denom = (sc->divide) ? dh1 * dh2 * dw1 * dw2 : 1; + denom = (sc->divide) ? dh1 * (int64_t)dh2 * dw1 * dw2 : 1; for (i = 0; i < 32; i++) { rowcount = 0; @@ -245,7 +245,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *picref) } } - denom = (sc->divide) ? 1 : dh1 * dh2 * dw1 * dw2; + denom = (sc->divide) ? 1 : dh1 * (int64_t)dh2 * dw1 * dw2; for (i = 0; i < ELEMENT_COUNT; i++) { const ElemCat* elemcat = elements[i]; -- cgit v1.2.3