From 9dc0bac9719a7b4c926d0c928ea8684e005f979d Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <michaelni@gmx.at>
Date: Thu, 2 Jul 2015 18:42:05 +0200
Subject: avcodec/motion_est_template: Fix undefined shifts in CHECK_MV()

Fixes:asan_heap-oob_4d5bb0_2295_cov_3374722539_hotel_california_ra5.1_640x480_30s.rmvb

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
---
 libavcodec/motion_est_template.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'libavcodec/motion_est_template.c')

diff --git a/libavcodec/motion_est_template.c b/libavcodec/motion_est_template.c
index cb8780160b..eb84d5d84d 100644
--- a/libavcodec/motion_est_template.c
+++ b/libavcodec/motion_est_template.c
@@ -358,8 +358,8 @@ static int qpel_motion_search(MpegEncContext * s,
 
 #define CHECK_MV(x,y)\
 {\
-    const unsigned key = ((y)<<ME_MAP_MV_BITS) + (x) + map_generation;\
-    const int index= (((y)<<ME_MAP_SHIFT) + (x))&(ME_MAP_SIZE-1);\
+    const unsigned key = ((unsigned)(y)<<ME_MAP_MV_BITS) + (x) + map_generation;\
+    const int index= (((unsigned)(y)<<ME_MAP_SHIFT) + (x))&(ME_MAP_SIZE-1);\
     av_assert2((x) >= xmin);\
     av_assert2((x) <= xmax);\
     av_assert2((y) >= ymin);\
@@ -368,7 +368,7 @@ static int qpel_motion_search(MpegEncContext * s,
         d= cmp(s, x, y, 0, 0, size, h, ref_index, src_index, cmpf, chroma_cmpf, flags);\
         map[index]= key;\
         score_map[index]= d;\
-        d += (mv_penalty[((x)<<shift)-pred_x] + mv_penalty[((y)<<shift)-pred_y])*penalty_factor;\
+        d += (mv_penalty[((x)*(1<<shift))-pred_x] + mv_penalty[((y)*(1<<shift))-pred_y])*penalty_factor;\
         COPY3_IF_LT(dmin, d, best[0], x, best[1], y)\
     }\
 }
-- 
cgit v1.2.3