diff options
| author | John Brooks <john.brooks@bluecherry.net> | 2011-12-12 17:04:14 -0700 |
|---|---|---|
| committer | Anton Khirnov <anton@khirnov.net> | 2012-01-21 08:28:08 +0100 |
| commit | 9b6aafba6c06ef62783dd5e9c5ed668f3a095128 (patch) | |
| tree | 3a02ccf7a244c2326bdfe49bf1b5a853b2281cfb /libavcodec/mpegvideo.c | |
| parent | 5d95112d926760046bc0b07aeda5cfc766ba54a6 (diff) | |
mpegvideo: fix invalid memory access for small video dimensions
When either video dimension is only one macroblock, subtractions
based on v_edge_pos and the macroblock size may be negative. In
that situation, an unsigned comparison isn't sufficent to test for
MV overruns, because a limit of (unsigned)-1 will let any other
value pass.
Signed-off-by: Anton Khirnov <anton@khirnov.net>
Diffstat (limited to 'libavcodec/mpegvideo.c')
| -rw-r--r-- | libavcodec/mpegvideo.c | 12 |
1 files changed, 6 insertions, 6 deletions
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c index f711d36aec..50e6ad6da4 100644 --- a/libavcodec/mpegvideo.c +++ b/libavcodec/mpegvideo.c @@ -1843,8 +1843,8 @@ static inline int hpel_motion_lowres(MpegEncContext *s, src += src_y * stride + src_x; - if ((unsigned)src_x > h_edge_pos - (!!sx) - w || - (unsigned)src_y > (v_edge_pos >> field_based) - (!!sy) - h) { + if ((unsigned)src_x > FFMAX( h_edge_pos - (!!sx) - w, 0) || + (unsigned)src_y > FFMAX((v_edge_pos >> field_based) - (!!sy) - h, 0)) { s->dsp.emulated_edge_mc(s->edge_emu_buffer, src, s->linesize, w + 1, (h + 1) << field_based, src_x, src_y << field_based, @@ -1928,8 +1928,8 @@ static av_always_inline void mpeg_motion_lowres(MpegEncContext *s, ptr_cb = ref_picture[1] + uvsrc_y * uvlinesize + uvsrc_x; ptr_cr = ref_picture[2] + uvsrc_y * uvlinesize + uvsrc_x; - if ((unsigned) src_x > h_edge_pos - (!!sx) - 2 * block_s || - (unsigned) src_y > (v_edge_pos >> field_based) - (!!sy) - h) { + if ((unsigned) src_x > FFMAX( h_edge_pos - (!!sx) - 2 * block_s, 0) || + (unsigned) src_y > FFMAX((v_edge_pos >> field_based) - (!!sy) - h, 0)) { s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr_y, s->linesize, 17, 17 + field_based, src_x, src_y << field_based, h_edge_pos, @@ -2011,8 +2011,8 @@ static inline void chroma_4mv_motion_lowres(MpegEncContext *s, offset = src_y * s->uvlinesize + src_x; ptr = ref_picture[1] + offset; if (s->flags & CODEC_FLAG_EMU_EDGE) { - if ((unsigned) src_x > h_edge_pos - (!!sx) - block_s || - (unsigned) src_y > v_edge_pos - (!!sy) - block_s) { + if ((unsigned) src_x > FFMAX(h_edge_pos - (!!sx) - block_s, 0) || + (unsigned) src_y > FFMAX(v_edge_pos - (!!sy) - block_s, 0)) { s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr, s->uvlinesize, 9, 9, src_x, src_y, h_edge_pos, v_edge_pos); ptr = s->edge_emu_buffer; |