quantizer overflow warning + AAN dct fix

Originally committed as revision 3754 to svn://svn.ffmpeg.org/ffmpeg/trunk

1 files changed, 9 insertions, 1 deletions

diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index 2b0b315c5c..0190c91b5d 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -164,7 +164,15 @@ static void convert_matrix(DSPContext *dsp, int (*qmat)[64], uint16_t (*qmat16)[
         }
         
         for(i=intra; i<64; i++){
-            while(((8191LL * qmat[qscale][i]) >> shift) > INT_MAX){ 
+            int64_t max= 8191;
+            if (dsp->fdct == fdct_ifast
+#ifndef FAAN_POSTSCALE
+                   || dsp->fdct == ff_faandct
+#endif
+                   ) {
+                max= (8191LL*aanscales[i]) >> 14;
+            }
+            while(((max * qmat[qscale][i]) >> shift) > INT_MAX){ 
                 shift++;
             }
         }