eac3dec: use 16-bit pre-mantissas instead of 24-bit in AHT decoding. it is

simpler and also fixes a bug in GAQ dequantization.

Originally committed as revision 18860 to svn://svn.ffmpeg.org/ffmpeg/trunk

4 files changed, 32 insertions, 31 deletions

diff --git a/libavcodec/ac3dec.c b/libavcodec/ac3dec.c
index bc39fb35b9..c176cb3d76 100644
--- a/libavcodec/ac3dec.c
+++ b/libavcodec/ac3dec.c
@@ -568,7 +568,7 @@ static void decode_transform_coeffs_ch(AC3DecodeContext *s, int blk, int ch,
         if (!blk)
             ff_eac3_decode_transform_coeffs_aht_ch(s, ch);
         for (bin = s->start_freq[ch]; bin < s->end_freq[ch]; bin++) {
-            s->fixed_coeffs[ch][bin] = s->pre_mantissa[ch][bin][blk] >> s->dexps[ch][bin];
+            s->fixed_coeffs[ch][bin] = (s->pre_mantissa[ch][bin][blk] << 8) >> s->dexps[ch][bin];
         }
     }
 }
diff --git a/libavcodec/ac3dec_data.c b/libavcodec/ac3dec_data.c
index 2a3719ea07..b9f98f904e 100644
--- a/libavcodec/ac3dec_data.c
+++ b/libavcodec/ac3dec_data.c
@@ -87,19 +87,18 @@ const int16_t ff_eac3_gaq_remap_2_4_a[9][2] = {
 /**
  * Table E3.6, Gk=2 & Gk=4, B
  * Large mantissa inverse quantization, negative mantissa remapping offsets
- * Table values from the spec are right-shifted by 8 to simplify calculations.
  * ff_eac3_gaq_remap_3_4_b[hebap-8][Gk=2,4]
  */
-const int8_t ff_eac3_gaq_remap_2_4_b[9][2] = {
-    { -22,  -5 },
-    { -46, -20 },
-    { -56, -26 },
-    { -60, -29 },
-    { -62, -31 },
-    { -63, -32 },
-    { -64, -32 },
-    { -64, -32 },
-    { -64, -32 },
+const int16_t ff_eac3_gaq_remap_2_4_b[9][2] = {
+    {  -5461, -1170 },
+    { -11703, -4915 },
+    { -14199, -6606 },
+    { -15327, -7412 },
+    { -15864, -7805 },
+    { -16126, -7999 },
+    { -16255, -8096 },
+    { -16320, -8144 },
+    { -16352, -8168 }
 };
 
 static const int16_t vq_hebap1[4][6] = {
diff --git a/libavcodec/ac3dec_data.h b/libavcodec/ac3dec_data.h
index 486d04a40b..c9bb58686b 100644
--- a/libavcodec/ac3dec_data.h
+++ b/libavcodec/ac3dec_data.h
@@ -29,7 +29,7 @@ extern const uint8_t ff_eac3_hebap_tab[64];
 extern const uint8_t ff_eac3_bits_vs_hebap[20];
 extern const int16_t ff_eac3_gaq_remap_1[12];
 extern const int16_t ff_eac3_gaq_remap_2_4_a[9][2];
-extern const int8_t  ff_eac3_gaq_remap_2_4_b[9][2];
+extern const int16_t ff_eac3_gaq_remap_2_4_b[9][2];
 
 extern const int16_t (* const ff_eac3_mantissa_vq[8])[6];
 extern const uint8_t ff_eac3_frm_expstr[32][6];
diff --git a/libavcodec/eac3dec.c b/libavcodec/eac3dec.c
index 8af729dc22..f78c50b987 100644
--- a/libavcodec/eac3dec.c
+++ b/libavcodec/eac3dec.c
@@ -66,18 +66,18 @@ typedef enum {
 
 #define EAC3_SR_CODE_REDUCED  3
 
-/** lrint(M_SQRT2*cos(2*M_PI/12)*(1<<23)) */
-#define COEFF_0 10273905LL
+/** lrint(M_SQRT2*cos(2*M_PI/12)*(1<<15)) */
+#define COEFF_0 40132
 
-/** lrint(M_SQRT2*cos(0*M_PI/12)*(1<<23)) = lrint(M_SQRT2*(1<<23)) */
-#define COEFF_1 11863283LL
+/** lrint(M_SQRT2*cos(0*M_PI/12)*(1<<15)) = lrint(M_SQRT2*(1<<15)) */
+#define COEFF_1 46341
 
-/** lrint(M_SQRT2*cos(5*M_PI/12)*(1<<23)) */
-#define COEFF_2  3070444LL
+/** lrint(M_SQRT2*cos(5*M_PI/12)*(1<<15)) */
+#define COEFF_2 11994
 
 /**
  * Calculate 6-point IDCT of the pre-mantissas.
- * All calculations are 24-bit fixed-point.
+ * All calculations are 16-bit fixed-point.
  */
 static void idct6(int pre_mant[6])
 {
@@ -86,9 +86,9 @@ static void idct6(int pre_mant[6])
 
     odd1 = pre_mant[1] - pre_mant[3] - pre_mant[5];
 
-    even2 = ( pre_mant[2]                * COEFF_0) >> 23;
-    tmp   = ( pre_mant[4]                * COEFF_1) >> 23;
-    odd0  = ((pre_mant[1] + pre_mant[5]) * COEFF_2) >> 23;
+    even2 = ( pre_mant[2]                * COEFF_0) >> 15;
+    tmp   = ( pre_mant[4]                * COEFF_1) >> 15;
+    odd0  = ((pre_mant[1] + pre_mant[5]) * COEFF_2) >> 15;
 
     even0 = pre_mant[0] + (tmp >> 1);
     even1 = pre_mant[0] - tmp;
@@ -155,13 +155,13 @@ void ff_eac3_decode_transform_coeffs_aht_ch(AC3DecodeContext *s, int ch)
         if (!hebap) {
             /* zero-mantissa dithering */
             for (blk = 0; blk < 6; blk++) {
-                s->pre_mantissa[ch][bin][blk] = (av_lfg_get(&s->dith_state) & 0x7FFFFF) - 0x400000;
+                s->pre_mantissa[ch][bin][blk] = (av_lfg_get(&s->dith_state) & 0x7FFF) - 0x4000;
             }
         } else if (hebap < 8) {
             /* Vector Quantization */
             int v = get_bits(gbc, bits);
             for (blk = 0; blk < 6; blk++) {
-                s->pre_mantissa[ch][bin][blk] = ff_eac3_mantissa_vq[hebap][v][blk] << 8;
+                s->pre_mantissa[ch][bin][blk] = ff_eac3_mantissa_vq[hebap][v][blk];
             }
         } else {
             /* Gain Adaptive Quantization */
@@ -175,22 +175,24 @@ void ff_eac3_decode_transform_coeffs_aht_ch(AC3DecodeContext *s, int ch)
 
             for (blk = 0; blk < 6; blk++) {
                 int mant = get_sbits(gbc, gbits);
-                if (mant == -(1 << (gbits-1))) {
+                if (log_gain > 0 && mant == -(1 << (gbits-1))) {
                     /* large mantissa */
                     int b;
-                    mant = get_sbits(gbc, bits-2+log_gain) << (26-log_gain-bits);
+                    int mbits = bits - (2 - log_gain);
+                    mant = get_sbits(gbc, mbits);
+                    mant <<= (15 - (mbits - 1));
                     /* remap mantissa value to correct for asymmetric quantization */
                     if (mant >= 0)
-                        b = 32768 >> (log_gain+8);
+                        b = 32768 >> log_gain;
                     else
                         b = ff_eac3_gaq_remap_2_4_b[hebap-8][log_gain-1];
-                    mant += (ff_eac3_gaq_remap_2_4_a[hebap-8][log_gain-1] * (mant>>8) + b) >> 7;
+                    mant += ((ff_eac3_gaq_remap_2_4_a[hebap-8][log_gain-1] * mant) >> 15) + b;
                 } else {
                     /* small mantissa, no GAQ, or Gk=1 */
-                    mant <<= 24 - bits;
+                    mant <<= 15 - (bits-1);
                     if (!log_gain) {
                         /* remap mantissa value for no GAQ or Gk=1 */
-                        mant += (ff_eac3_gaq_remap_1[hebap-8] * (mant>>8)) >> 7;
+                        mant += (ff_eac3_gaq_remap_1[hebap-8] * mant) >> 15;
                     }
                 }
                 s->pre_mantissa[ch][bin][blk] = mant;