From e71ca21f308432cac3deaabe522ac1b856471162 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <michael@niedermayer.cc>
Date: Fri, 10 Jul 2015 13:21:18 +0200
Subject: avcodec/motion_est_template: Fix undefined behavior in
 small_diamond_search()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes: asan_heap-oob_394322e_138_cov_4265020547_CVPCMNL1_SVA_C.264

Found-by: Samuel Groß, Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavcodec/motion_est_template.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'libavcodec/motion_est_template.c')

diff --git a/libavcodec/motion_est_template.c b/libavcodec/motion_est_template.c
index eb84d5d84d..25bab9ddfb 100644
--- a/libavcodec/motion_est_template.c
+++ b/libavcodec/motion_est_template.c
@@ -426,8 +426,8 @@ static av_always_inline int small_diamond_search(MpegEncContext * s, int *best,
     chroma_cmpf = s->mecc.me_cmp[size + 1];
 
     { /* ensure that the best point is in the MAP as h/qpel refinement needs it */
-        const unsigned key = (best[1]<<ME_MAP_MV_BITS) + best[0] + map_generation;
-        const int index= ((best[1]<<ME_MAP_SHIFT) + best[0])&(ME_MAP_SIZE-1);
+        const unsigned key = ((unsigned)best[1]<<ME_MAP_MV_BITS) + best[0] + map_generation;
+        const int index= (((unsigned)best[1]<<ME_MAP_SHIFT) + best[0])&(ME_MAP_SIZE-1);
         if(map[index]!=key){ //this will be executed only very rarey
             score_map[index]= cmp(s, best[0], best[1], 0, 0, size, h, ref_index, src_index, cmpf, chroma_cmpf, flags);
             map[index]= key;
-- 
cgit v1.2.3