From 3ef65fd4d1d0cdfa0b60351c719bef93d3664ea2 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
Date: Fri, 26 Mar 2021 18:18:05 +0100
Subject: avcodec/motion_est: Fix invalid left shift of negative numbers

Affected many FATE-tests.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
---
 libavcodec/motion_est_template.c | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

(limited to 'libavcodec/motion_est_template.c')

diff --git a/libavcodec/motion_est_template.c b/libavcodec/motion_est_template.c
index d3b2f97744..6ab0ea13dc 100644
--- a/libavcodec/motion_est_template.c
+++ b/libavcodec/motion_est_template.c
@@ -82,7 +82,7 @@ static int hpel_motion_search(MpegEncContext * s,
     if (mx > xmin && mx < xmax &&
         my > ymin && my < ymax) {
         int d= dmin;
-        const int index= (my<<ME_MAP_SHIFT) + mx;
+        const int index = my * (1 << ME_MAP_SHIFT) + mx;
         const int t= score_map[(index-(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)]
                      + (mv_penalty[bx   - pred_x] + mv_penalty[by-2 - pred_y])*c->penalty_factor;
         const int l= score_map[(index- 1               )&(ME_MAP_SIZE-1)]
@@ -95,13 +95,13 @@ static int hpel_motion_search(MpegEncContext * s,
 #if defined(ASSERT_LEVEL) && ASSERT_LEVEL > 1
         unsigned key;
         unsigned map_generation= c->map_generation;
-        key= ((my-1)<<ME_MAP_MV_BITS) + (mx) + map_generation;
+        key = (my - 1) * (1 << ME_MAP_MV_BITS) + (mx) + map_generation;
         av_assert2(c->map[(index-(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)] == key);
-        key= ((my+1)<<ME_MAP_MV_BITS) + (mx) + map_generation;
+        key = (my + 1) * (1 << ME_MAP_MV_BITS) + (mx) + map_generation;
         av_assert2(c->map[(index+(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)] == key);
-        key= ((my)<<ME_MAP_MV_BITS) + (mx+1) + map_generation;
+        key = (my) * (1 << ME_MAP_MV_BITS) + (mx + 1) + map_generation;
         av_assert2(c->map[(index+1)&(ME_MAP_SIZE-1)] == key);
-        key= ((my)<<ME_MAP_MV_BITS) + (mx-1) + map_generation;
+        key = (my) * (1 << ME_MAP_MV_BITS) + (mx - 1) + map_generation;
         av_assert2(c->map[(index-1)&(ME_MAP_SIZE-1)] == key);
 #endif
         if(t<=b){
@@ -246,7 +246,7 @@ static int qpel_motion_search(MpegEncContext * s,
         int bx=4*mx, by=4*my;
         int d= dmin;
         int i, nx, ny;
-        const int index= (my<<ME_MAP_SHIFT) + mx;
+        const int index = my * (1 << ME_MAP_SHIFT) + mx;
         const int t= score_map[(index-(1<<ME_MAP_SHIFT)  )&(ME_MAP_SIZE-1)];
         const int l= score_map[(index- 1                 )&(ME_MAP_SIZE-1)];
         const int r= score_map[(index+ 1                 )&(ME_MAP_SIZE-1)];
@@ -299,7 +299,8 @@ static int qpel_motion_search(MpegEncContext * s,
             const int cy2= b + t - 2*c;
             int cxy;
 
-            if(map[(index-(1<<ME_MAP_SHIFT)-1)&(ME_MAP_SIZE-1)] == ((my-1)<<ME_MAP_MV_BITS) + (mx-1) + map_generation){
+            if (map[(index - (1 << ME_MAP_SHIFT) - 1) & (ME_MAP_SIZE - 1)] ==
+                (my - 1) * (1 << ME_MAP_MV_BITS) + (mx - 1) + map_generation) {
                 tl= score_map[(index-(1<<ME_MAP_SHIFT)-1)&(ME_MAP_SIZE-1)];
             }else{
                 tl= cmp(s, mx-1, my-1, 0, 0, size, h, ref_index, src_index, cmpf, chroma_cmpf, flags);//FIXME wrong if chroma me is different
@@ -328,7 +329,7 @@ static int qpel_motion_search(MpegEncContext * s,
                     for(i=0; i<8; i++){
                         if(score < best[i]){
                             memmove(&best[i+1], &best[i], sizeof(int)*(7-i));
-                            memmove(&best_pos[i+1][0], &best_pos[i][0], sizeof(int)*2*(7-i));
+                            memmove(best_pos[i + 1], best_pos[i], sizeof(best_pos[0]) * (7 - i));
                             best[i]= score;
                             best_pos[i][0]= nx + 4*mx;
                             best_pos[i][1]= ny + 4*my;
-- 
cgit v1.2.3