Merge remote-tracking branch 'qatar/master'

* qatar/master:
  configure: add --optflags option
  build: move documentation rules to doc/Makefile
  build: move test rules to tests/Makefile
  ac3enc: remove unneeded local variable in asym_quant()
  ac3enc: remove a branch in asym_quant() by doing 2 shifts
  ac3enc: avoid masking output in asym_quant() by using signed values for quantized mantissas.
  H.264: fix 4:4:4 + deblocking + 8x8dct + cavlc + MBAFF
  H.264: fix 4:4:4 + deblocking + MBAFF
  H.264: fix 4:4:4 cropping warning
  H.264: reference the correct SPS in decode_scaling_matrices
  H.264: fix bug in lossless 4:4:4 decoding

Conflicts:
	Makefile

Merged-by: Michael Niedermayer <michaelni@gmx.at>

5 files changed, 50 insertions, 41 deletions

diff --git a/libavcodec/ac3enc.c b/libavcodec/ac3enc.c
index 6ee8a7ac15..749967fedb 100644
--- a/libavcodec/ac3enc.c
+++ b/libavcodec/ac3enc.c
@@ -46,7 +46,7 @@
 #include "eac3enc.h"
 
 typedef struct AC3Mant {
-    uint16_t *qmant1_ptr, *qmant2_ptr, *qmant4_ptr; ///< mantissa pointers for bap=1,2,4
+    int16_t *qmant1_ptr, *qmant2_ptr, *qmant4_ptr; ///< mantissa pointers for bap=1,2,4
     int mant1_cnt, mant2_cnt, mant4_cnt;    ///< mantissa counts for bap=1,2,4
 } AC3Mant;
 
@@ -1123,20 +1123,14 @@ static inline int sym_quant(int c, int e, int levels)
  */
 static inline int asym_quant(int c, int e, int qbits)
 {
-    int lshift, m, v;
+    int m;
 
-    lshift = e + qbits - 24;
-    if (lshift >= 0)
-        v = c << lshift;
-    else
-        v = c >> (-lshift);
-    /* rounding */
-    v = (v + 1) >> 1;
+    c = (((c << e) >> (24 - qbits)) + 1) >> 1;
     m = (1 << (qbits-1));
-    if (v >= m)
-        v = m - 1;
-    av_assert2(v >= -m);
-    return v & ((1 << qbits)-1);
+    if (c >= m)
+        c = m - 1;
+    av_assert2(c >= -m);
+    return c;
 }
 
 
@@ -1145,7 +1139,7 @@ static inline int asym_quant(int c, int e, int qbits)
  */
 static void quantize_mantissas_blk_ch(AC3Mant *s, int32_t *fixed_coef,
                                       uint8_t *exp, uint8_t *bap,
-                                      uint16_t *qmant, int start_freq,
+                                      int16_t *qmant, int start_freq,
                                       int end_freq)
 {
     int i;
@@ -1497,14 +1491,14 @@ static void output_audio_block(AC3EncodeContext *s, int blk)
             q = block->qmant[ch][i];
             b = s->ref_bap[ch][blk][i];
             switch (b) {
-            case 0:                                         break;
-            case 1: if (q != 128) put_bits(&s->pb,   5, q); break;
-            case 2: if (q != 128) put_bits(&s->pb,   7, q); break;
-            case 3:               put_bits(&s->pb,   3, q); break;
-            case 4: if (q != 128) put_bits(&s->pb,   7, q); break;
-            case 14:              put_bits(&s->pb,  14, q); break;
-            case 15:              put_bits(&s->pb,  16, q); break;
-            default:              put_bits(&s->pb, b-1, q); break;
+            case 0:                                          break;
+            case 1: if (q != 128) put_bits (&s->pb,   5, q); break;
+            case 2: if (q != 128) put_bits (&s->pb,   7, q); break;
+            case 3:               put_sbits(&s->pb,   3, q); break;
+            case 4: if (q != 128) put_bits (&s->pb,   7, q); break;
+            case 14:              put_sbits(&s->pb,  14, q); break;
+            case 15:              put_sbits(&s->pb,  16, q); break;
+            default:              put_sbits(&s->pb, b-1, q); break;
             }
         }
         if (ch == CPL_CH)
diff --git a/libavcodec/ac3enc.h b/libavcodec/ac3enc.h
index 2d04f26303..490cb2b26d 100644
--- a/libavcodec/ac3enc.h
+++ b/libavcodec/ac3enc.h
@@ -212,7 +212,7 @@ typedef struct AC3EncodeContext {
     int16_t *psd_buffer;
     int16_t *band_psd_buffer;
     int16_t *mask_buffer;
-    uint16_t *qmant_buffer;
+    int16_t *qmant_buffer;
     uint8_t *cpl_coord_exp_buffer;
     uint8_t *cpl_coord_mant_buffer;
 
diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index aed5219649..dd9f425c3b 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -3303,8 +3303,8 @@ static void loop_filter(H264Context *h, int start_x, int end_x){
                     uvlinesize = h->mb_uvlinesize = s->uvlinesize * 2;
                     if(mb_y&1){ //FIXME move out of this function?
                         dest_y -= s->linesize*15;
-                        dest_cb-= s->uvlinesize*7;
-                        dest_cr-= s->uvlinesize*7;
+                        dest_cb-= s->uvlinesize*((8 << CHROMA444)-1);
+                        dest_cr-= s->uvlinesize*((8 << CHROMA444)-1);
                     }
                 } else {
                     linesize   = h->mb_linesize   = s->linesize;
diff --git a/libavcodec/h264_loopfilter.c b/libavcodec/h264_loopfilter.c
index fd1fb5f662..46abc54c49 100644
--- a/libavcodec/h264_loopfilter.c
+++ b/libavcodec/h264_loopfilter.c
@@ -393,10 +393,10 @@ static av_always_inline void filter_mb_dir(H264Context *h, int mb_x, int mb_y, u
                     AV_WN64A(bS, 0x0003000300030003ULL);
                 } else {
                     if(!CABAC && IS_8x8DCT(s->current_picture.mb_type[mbn_xy])){
-                        bS[0]= 1+((h->cbp_table[mbn_xy] & 4)||h->non_zero_count_cache[scan8[0]+0]);
-                        bS[1]= 1+((h->cbp_table[mbn_xy] & 4)||h->non_zero_count_cache[scan8[0]+1]);
-                        bS[2]= 1+((h->cbp_table[mbn_xy] & 8)||h->non_zero_count_cache[scan8[0]+2]);
-                        bS[3]= 1+((h->cbp_table[mbn_xy] & 8)||h->non_zero_count_cache[scan8[0]+3]);
+                        bS[0]= 1+((h->cbp_table[mbn_xy] & 0x4000)||h->non_zero_count_cache[scan8[0]+0]);
+                        bS[1]= 1+((h->cbp_table[mbn_xy] & 0x4000)||h->non_zero_count_cache[scan8[0]+1]);
+                        bS[2]= 1+((h->cbp_table[mbn_xy] & 0x8000)||h->non_zero_count_cache[scan8[0]+2]);
+                        bS[3]= 1+((h->cbp_table[mbn_xy] & 0x8000)||h->non_zero_count_cache[scan8[0]+3]);
                     }else{
                     const uint8_t *mbn_nnz = h->non_zero_count[mbn_xy] + 3*4;
                     int i;
@@ -635,7 +635,7 @@ void ff_h264_filter_mb( H264Context *h, int mb_x, int mb_y, uint8_t *img_y, uint
                 else{
                     bS[i] = 1 + !!(h->non_zero_count_cache[12+8*(i>>1)] |
                          ((!h->pps.cabac && IS_8x8DCT(mbn_type)) ?
-                            (h->cbp_table[mbn_xy] & ((MB_FIELD ? (i&2) : (mb_y&1)) ? 8 : 2))
+                            (h->cbp_table[mbn_xy] & (((MB_FIELD ? (i&2) : (mb_y&1)) ? 8 : 2) << 12))
                                                                        :
                             h->non_zero_count[mbn_xy][ off[i] ]));
                 }
@@ -663,19 +663,33 @@ void ff_h264_filter_mb( H264Context *h, int mb_x, int mb_y, uint8_t *img_y, uint
             filter_mb_mbaff_edgev ( h, img_y                ,   linesize, bS  , 1, qp [0] );
             filter_mb_mbaff_edgev ( h, img_y  + 8*  linesize,   linesize, bS+4, 1, qp [1] );
             if (chroma){
-                filter_mb_mbaff_edgecv( h, img_cb,                uvlinesize, bS  , 1, bqp[0] );
-                filter_mb_mbaff_edgecv( h, img_cb + 4*uvlinesize, uvlinesize, bS+4, 1, bqp[1] );
-                filter_mb_mbaff_edgecv( h, img_cr,                uvlinesize, bS  , 1, rqp[0] );
-                filter_mb_mbaff_edgecv( h, img_cr + 4*uvlinesize, uvlinesize, bS+4, 1, rqp[1] );
+                if (CHROMA444) {
+                    filter_mb_mbaff_edgev ( h, img_cb,                uvlinesize, bS  , 1, bqp[0] );
+                    filter_mb_mbaff_edgev ( h, img_cb + 8*uvlinesize, uvlinesize, bS+4, 1, bqp[1] );
+                    filter_mb_mbaff_edgev ( h, img_cr,                uvlinesize, bS  , 1, rqp[0] );
+                    filter_mb_mbaff_edgev ( h, img_cr + 8*uvlinesize, uvlinesize, bS+4, 1, rqp[1] );
+                }else{
+                    filter_mb_mbaff_edgecv( h, img_cb,                uvlinesize, bS  , 1, bqp[0] );
+                    filter_mb_mbaff_edgecv( h, img_cb + 4*uvlinesize, uvlinesize, bS+4, 1, bqp[1] );
+                    filter_mb_mbaff_edgecv( h, img_cr,                uvlinesize, bS  , 1, rqp[0] );
+                    filter_mb_mbaff_edgecv( h, img_cr + 4*uvlinesize, uvlinesize, bS+4, 1, rqp[1] );
+                }
             }
         }else{
             filter_mb_mbaff_edgev ( h, img_y              , 2*  linesize, bS  , 2, qp [0] );
             filter_mb_mbaff_edgev ( h, img_y  +   linesize, 2*  linesize, bS+1, 2, qp [1] );
             if (chroma){
-                filter_mb_mbaff_edgecv( h, img_cb,              2*uvlinesize, bS  , 2, bqp[0] );
-                filter_mb_mbaff_edgecv( h, img_cb + uvlinesize, 2*uvlinesize, bS+1, 2, bqp[1] );
-                filter_mb_mbaff_edgecv( h, img_cr,              2*uvlinesize, bS  , 2, rqp[0] );
-                filter_mb_mbaff_edgecv( h, img_cr + uvlinesize, 2*uvlinesize, bS+1, 2, rqp[1] );
+                if (CHROMA444) {
+                    filter_mb_mbaff_edgev ( h, img_cb,              2*uvlinesize, bS  , 2, bqp[0] );
+                    filter_mb_mbaff_edgev ( h, img_cb + uvlinesize, 2*uvlinesize, bS+1, 2, bqp[1] );
+                    filter_mb_mbaff_edgev ( h, img_cr,              2*uvlinesize, bS  , 2, rqp[0] );
+                    filter_mb_mbaff_edgev ( h, img_cr + uvlinesize, 2*uvlinesize, bS+1, 2, rqp[1] );
+                }else{
+                    filter_mb_mbaff_edgecv( h, img_cb,              2*uvlinesize, bS  , 2, bqp[0] );
+                    filter_mb_mbaff_edgecv( h, img_cb + uvlinesize, 2*uvlinesize, bS+1, 2, bqp[1] );
+                    filter_mb_mbaff_edgecv( h, img_cr,              2*uvlinesize, bS  , 2, rqp[0] );
+                    filter_mb_mbaff_edgecv( h, img_cr + uvlinesize, 2*uvlinesize, bS+1, 2, rqp[1] );
+                }
             }
         }
     }
diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c
index ddfe1d2c64..61fb12ce0c 100644
--- a/libavcodec/h264_ps.c
+++ b/libavcodec/h264_ps.c
@@ -281,12 +281,12 @@ static void decode_scaling_matrices(H264Context *h, SPS *sps, PPS *pps, int is_s
         decode_scaling_list(h,scaling_matrix4[5],16,default_scaling4[1],scaling_matrix4[4]); // Inter, Cb
         if(is_sps || pps->transform_8x8_mode){
             decode_scaling_list(h,scaling_matrix8[0],64,default_scaling8[0],fallback[2]);  // Intra, Y
-            if(h->sps.chroma_format_idc == 3){
+            if(sps->chroma_format_idc == 3){
                 decode_scaling_list(h,scaling_matrix8[1],64,default_scaling8[0],scaling_matrix8[0]);  // Intra, Cr
                 decode_scaling_list(h,scaling_matrix8[2],64,default_scaling8[0],scaling_matrix8[1]);  // Intra, Cb
             }
             decode_scaling_list(h,scaling_matrix8[3],64,default_scaling8[1],fallback[3]);  // Inter, Y
-            if(h->sps.chroma_format_idc == 3){
+            if(sps->chroma_format_idc == 3){
                 decode_scaling_list(h,scaling_matrix8[4],64,default_scaling8[1],scaling_matrix8[3]);  // Inter, Cr
                 decode_scaling_list(h,scaling_matrix8[5],64,default_scaling8[1],scaling_matrix8[4]);  // Inter, Cb
             }
@@ -396,6 +396,7 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){
 #endif
     sps->crop= get_bits1(&s->gb);
     if(sps->crop){
+        int crop_limit = sps->chroma_format_idc == 3 ? 16 : 8;
         sps->crop_left  = get_ue_golomb(&s->gb);
         sps->crop_right = get_ue_golomb(&s->gb);
         sps->crop_top   = get_ue_golomb(&s->gb);
@@ -403,7 +404,7 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){
         if(sps->crop_left || sps->crop_top){
             av_log(h->s.avctx, AV_LOG_ERROR, "insane cropping not completely supported, this could look slightly wrong ...\n");
         }
-        if(sps->crop_right >= (8<<CHROMA444) || sps->crop_bottom >= (8<<CHROMA444)){
+        if(sps->crop_right >= crop_limit || sps->crop_bottom >= crop_limit){
             av_log(h->s.avctx, AV_LOG_ERROR, "brainfart cropping not supported, this could look slightly wrong ...\n");
         }
     }else{