From 2d66fc543b01995d6146fc132a778d3e722ca665 Mon Sep 17 00:00:00 2001 From: Clément Bœsch Date: Fri, 7 Dec 2012 00:36:29 +0100 Subject: lavfi/gradfun: fix rounding in MMX code. Current code divides before increasing precision. Also reduce upper bound for strength from 255 to 64. This will prevent an overflow in the SSSE3 and MMX filter_line code: delta is expressed as an u16 being shifted by 2 to the left. If it overflows, having a strength not above 64 will make sure that m is set to 0 (making the m*m*delta >> 14 expression void). A value above 64 should not make any sense unless gradfun is used as a blur filter. Signed-off-by: Anton Khirnov --- libavfilter/x86/vf_gradfun.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libavfilter/x86') diff --git a/libavfilter/x86/vf_gradfun.c b/libavfilter/x86/vf_gradfun.c index b4ca86c617..75c117a9a2 100644 --- a/libavfilter/x86/vf_gradfun.c +++ b/libavfilter/x86/vf_gradfun.c @@ -62,8 +62,8 @@ static void gradfun_filter_line_mmxext(uint8_t *dst, uint8_t *src, uint16_t *dc, "pminsw %%mm7, %%mm2 \n" // m = -max(0, 127-m) "pmullw %%mm2, %%mm2 \n" "paddw %%mm4, %%mm0 \n" // pix += dither - "pmulhw %%mm2, %%mm1 \n" "psllw $2, %%mm1 \n" // m = m*m*delta >> 14 + "pmulhw %%mm2, %%mm1 \n" "paddw %%mm1, %%mm0 \n" // pix += m "psraw $7, %%mm0 \n" "packuswb %%mm0, %%mm0 \n" -- cgit v1.2.3