From aacf6b3a2fd8bc8603e3deaa6e612ea03cf08707 Mon Sep 17 00:00:00 2001 From: John Brooks Date: Tue, 3 Jan 2012 23:27:35 -0700 Subject: rv34: fix invalid memory access for small video dimensions For small video dimensions calculations of the upper bound for pixel access may result in negative value. Using an unsigned comparison works only if the greater operand is non-negative. This is fixed by doing edge emulation explicitly for such conditions. Signed-off-by: Kostya Shishkov --- libavcodec/rv34.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'libavcodec/rv34.c') diff --git a/libavcodec/rv34.c b/libavcodec/rv34.c index db2e53aaa5..91d678876a 100644 --- a/libavcodec/rv34.c +++ b/libavcodec/rv34.c @@ -680,8 +680,9 @@ static inline void rv34_mc(RV34DecContext *r, const int block_type, srcY += src_y * s->linesize + src_x; srcU += uvsrc_y * s->uvlinesize + uvsrc_x; srcV += uvsrc_y * s->uvlinesize + uvsrc_x; - if( (unsigned)(src_x - !!lx*2) > s->h_edge_pos - !!lx*2 - (width <<3) - 4 - || (unsigned)(src_y - !!ly*2) > s->v_edge_pos - !!ly*2 - (height<<3) - 4){ + if(s->h_edge_pos - (width << 3) < 6 || s->v_edge_pos - (height << 3) < 6 || + (unsigned)(src_x - !!lx*2) > s->h_edge_pos - !!lx*2 - (width <<3) - 4 || + (unsigned)(src_y - !!ly*2) > s->v_edge_pos - !!ly*2 - (height<<3) - 4) { uint8_t *uvbuf = s->edge_emu_buffer + 22 * s->linesize; srcY -= 2 + 2*s->linesize; -- cgit v1.2.3