lavc/flac: Fix encoding and decoding with high lpc.

Based on an analysis by trac user lvqcl.

Fixes ticket #4421, reported by Chase Walker.

8 files changed, 81 insertions, 26 deletions

diff --git a/doc/decoders.texi b/doc/decoders.texi
index 01fca9fc55..68196cf111 100644
--- a/doc/decoders.texi
+++ b/doc/decoders.texi
@@ -83,6 +83,23 @@ Loud sounds are fully compressed.  Soft sounds are enhanced.
 
 @end table
 
+@section flac
+
+FLAC audio decoder.
+
+This decoder aims to implement the complete FLAC specification from Xiph.
+
+@subsection FLAC Decoder options
+
+@table @option
+
+@item -use_buggy_lpc
+The lavc FLAC encoder used to produce buggy streams with high lpc values
+(like the default value). This option allows to decode such streams
+correctly by using lavc's old buggy lpc logic for decoding.
+
+@end table
+
 @section ffwavesynth
 
 Internal wave synthetizer.
diff --git a/libavcodec/arm/flacdsp_init_arm.c b/libavcodec/arm/flacdsp_init_arm.c
index 82a807f8ba..564e3dc79b 100644
--- a/libavcodec/arm/flacdsp_init_arm.c
+++ b/libavcodec/arm/flacdsp_init_arm.c
@@ -27,6 +27,6 @@ void ff_flac_lpc_16_arm(int32_t *samples, const int coeffs[32], int order,
 av_cold void ff_flacdsp_init_arm(FLACDSPContext *c, enum AVSampleFormat fmt, int channels,
                                  int bps)
 {
-    if (CONFIG_FLAC_DECODER && bps <= 16)
-        c->lpc = ff_flac_lpc_16_arm;
+    if (CONFIG_FLAC_DECODER)
+        c->lpc16 = ff_flac_lpc_16_arm;
 }
diff --git a/libavcodec/flacdec.c b/libavcodec/flacdec.c
index 00b4726875..36d2928137 100644
--- a/libavcodec/flacdec.c
+++ b/libavcodec/flacdec.c
@@ -35,6 +35,7 @@
 
 #include "libavutil/avassert.h"
 #include "libavutil/crc.h"
+#include "libavutil/opt.h"
 #include "avcodec.h"
 #include "internal.h"
 #include "get_bits.h"
@@ -48,6 +49,7 @@
 
 
 typedef struct FLACContext {
+    AVClass *class;
     struct FLACStreaminfo flac_stream_info;
 
     AVCodecContext *avctx;                  ///< parent AVCodecContext
@@ -61,6 +63,7 @@ typedef struct FLACContext {
     int32_t *decoded[FLAC_MAX_CHANNELS];    ///< decoded samples
     uint8_t *decoded_buffer;
     unsigned int decoded_buffer_size;
+    int buggy_lpc;                          ///< use workaround for old lavc encoded files
 
     FLACDSPContext dsp;
 } FLACContext;
@@ -343,7 +346,13 @@ static int decode_subframe_lpc(FLACContext *s, int32_t *decoded, int pred_order,
     if ((ret = decode_residuals(s, decoded, pred_order)) < 0)
         return ret;
 
-    s->dsp.lpc(decoded, coeffs, pred_order, qlevel, s->blocksize);
+    if (   (    s->buggy_lpc && s->flac_stream_info.bps <= 16)
+        || (   !s->buggy_lpc && bps <= 16
+            && bps + coeff_prec + av_log2(pred_order) <= 32)) {
+        s->dsp.lpc16(decoded, coeffs, pred_order, qlevel, s->blocksize);
+    } else {
+        s->dsp.lpc32(decoded, coeffs, pred_order, qlevel, s->blocksize);
+    }
 
     return 0;
 }
@@ -605,6 +614,18 @@ static av_cold int flac_decode_close(AVCodecContext *avctx)
     return 0;
 }
 
+static const AVOption options[] = {
+{ "use_buggy_lpc", "emulate old buggy lavc behavior", offsetof(FLACContext, buggy_lpc), AV_OPT_TYPE_INT, {.i64 = 0 }, 0, 1, AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_AUDIO_PARAM },
+{ NULL },
+};
+
+static const AVClass flac_decoder_class = {
+    "FLAC decoder",
+    av_default_item_name,
+    options,
+    LIBAVUTIL_VERSION_INT,
+};
+
 AVCodec ff_flac_decoder = {
     .name           = "flac",
     .long_name      = NULL_IF_CONFIG_SMALL("FLAC (Free Lossless Audio Codec)"),
@@ -621,4 +642,5 @@ AVCodec ff_flac_decoder = {
                                                       AV_SAMPLE_FMT_S32,
                                                       AV_SAMPLE_FMT_S32P,
                                                       AV_SAMPLE_FMT_NONE },
+    .priv_class     = &flac_decoder_class,
 };
diff --git a/libavcodec/flacdsp.c b/libavcodec/flacdsp.c
index a83eb833b5..30b66484e8 100644
--- a/libavcodec/flacdsp.c
+++ b/libavcodec/flacdsp.c
@@ -88,13 +88,10 @@ static void flac_lpc_32_c(int32_t *decoded, const int coeffs[32],
 av_cold void ff_flacdsp_init(FLACDSPContext *c, enum AVSampleFormat fmt, int channels,
                              int bps)
 {
-    if (bps > 16) {
-        c->lpc            = flac_lpc_32_c;
-        c->lpc_encode     = flac_lpc_encode_c_32;
-    } else {
-        c->lpc            = flac_lpc_16_c;
-        c->lpc_encode     = flac_lpc_encode_c_16;
-    }
+    c->lpc16        = flac_lpc_16_c;
+    c->lpc32        = flac_lpc_32_c;
+    c->lpc16_encode = flac_lpc_encode_c_16;
+    c->lpc32_encode = flac_lpc_encode_c_32;
 
     switch (fmt) {
     case AV_SAMPLE_FMT_S32:
diff --git a/libavcodec/flacdsp.h b/libavcodec/flacdsp.h
index 417381cc70..f5cbd94724 100644
--- a/libavcodec/flacdsp.h
+++ b/libavcodec/flacdsp.h
@@ -25,10 +25,14 @@
 typedef struct FLACDSPContext {
     void (*decorrelate[4])(uint8_t **out, int32_t **in, int channels,
                            int len, int shift);
-    void (*lpc)(int32_t *samples, const int coeffs[32], int order,
-                int qlevel, int len);
-    void (*lpc_encode)(int32_t *res, const int32_t *smp, int len, int order,
-                       const int32_t coefs[32], int shift);
+    void (*lpc16)(int32_t *samples, const int coeffs[32], int order,
+                  int qlevel, int len);
+    void (*lpc32)(int32_t *samples, const int coeffs[32], int order,
+                  int qlevel, int len);
+    void (*lpc16_encode)(int32_t *res, const int32_t *smp, int len, int order,
+                         const int32_t coefs[32], int shift);
+    void (*lpc32_encode)(int32_t *res, const int32_t *smp, int len, int order,
+                         const int32_t coefs[32], int shift);
 } FLACDSPContext;
 
 void ff_flacdsp_init(FLACDSPContext *c, enum AVSampleFormat fmt, int channels, int bps);
diff --git a/libavcodec/flacenc.c b/libavcodec/flacenc.c
index bc6d00af7f..b6dc4d5c07 100644
--- a/libavcodec/flacenc.c
+++ b/libavcodec/flacenc.c
@@ -838,8 +838,13 @@ static int encode_residual_ch(FlacEncodeContext *s, int ch)
             order = av_clip(order, min_order - 1, max_order - 1);
             if (order == last_order)
                 continue;
-            s->flac_dsp.lpc_encode(res, smp, n, order+1, coefs[order],
-                                   shift[order]);
+            if (s->bps_code * 4 + s->options.lpc_coeff_precision + av_log2(order) <= 32) {
+                s->flac_dsp.lpc16_encode(res, smp, n, order+1, coefs[order],
+                                         shift[order]);
+            } else {
+                s->flac_dsp.lpc32_encode(res, smp, n, order+1, coefs[order],
+                                         shift[order]);
+            }
             bits[i] = find_subframe_rice_params(s, sub, order+1);
             if (bits[i] < bits[opt_index]) {
                 opt_index = i;
@@ -853,7 +858,11 @@ static int encode_residual_ch(FlacEncodeContext *s, int ch)
         opt_order = 0;
         bits[0]   = UINT32_MAX;
         for (i = min_order-1; i < max_order; i++) {
-            s->flac_dsp.lpc_encode(res, smp, n, i+1, coefs[i], shift[i]);
+            if (s->bps_code * 4 + s->options.lpc_coeff_precision + av_log2(i) <= 32) {
+                s->flac_dsp.lpc16_encode(res, smp, n, i+1, coefs[i], shift[i]);
+            } else {
+                s->flac_dsp.lpc32_encode(res, smp, n, i+1, coefs[i], shift[i]);
+            }
             bits[i] = find_subframe_rice_params(s, sub, i+1);
             if (bits[i] < bits[opt_order])
                 opt_order = i;
@@ -871,7 +880,11 @@ static int encode_residual_ch(FlacEncodeContext *s, int ch)
             for (i = last-step; i <= last+step; i += step) {
                 if (i < min_order-1 || i >= max_order || bits[i] < UINT32_MAX)
                     continue;
-                s->flac_dsp.lpc_encode(res, smp, n, i+1, coefs[i], shift[i]);
+                if (s->bps_code * 4 + s->options.lpc_coeff_precision + av_log2(i) <= 32) {
+                    s->flac_dsp.lpc32_encode(res, smp, n, i+1, coefs[i], shift[i]);
+                } else {
+                    s->flac_dsp.lpc16_encode(res, smp, n, i+1, coefs[i], shift[i]);
+                }
                 bits[i] = find_subframe_rice_params(s, sub, i+1);
                 if (bits[i] < bits[opt_order])
                     opt_order = i;
@@ -886,7 +899,11 @@ static int encode_residual_ch(FlacEncodeContext *s, int ch)
     for (i = 0; i < sub->order; i++)
         sub->coefs[i] = coefs[sub->order-1][i];
 
-    s->flac_dsp.lpc_encode(res, smp, n, sub->order, sub->coefs, sub->shift);
+    if (s->bps_code * 4 + s->options.lpc_coeff_precision + av_log2(opt_order) <= 32) {
+        s->flac_dsp.lpc16_encode(res, smp, n, sub->order, sub->coefs, sub->shift);
+    } else {
+        s->flac_dsp.lpc32_encode(res, smp, n, sub->order, sub->coefs, sub->shift);
+    }
 
     find_subframe_rice_params(s, sub, sub->order);
 
diff --git a/libavcodec/version.h b/libavcodec/version.h
index 8679ac6766..dd47af0f81 100644
--- a/libavcodec/version.h
+++ b/libavcodec/version.h
@@ -30,7 +30,7 @@
 
 #define LIBAVCODEC_VERSION_MAJOR 56
 #define LIBAVCODEC_VERSION_MINOR  39
-#define LIBAVCODEC_VERSION_MICRO 100
+#define LIBAVCODEC_VERSION_MICRO 101
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
                                                LIBAVCODEC_VERSION_MINOR, \
diff --git a/libavcodec/x86/flacdsp_init.c b/libavcodec/x86/flacdsp_init.c
index d04af4511d..e28c5c9322 100644
--- a/libavcodec/x86/flacdsp_init.c
+++ b/libavcodec/x86/flacdsp_init.c
@@ -85,8 +85,7 @@ av_cold void ff_flacdsp_init_x86(FLACDSPContext *c, enum AVSampleFormat fmt, int
         }
     }
     if (EXTERNAL_SSE4(cpu_flags)) {
-        if (bps > 16)
-            c->lpc = ff_flac_lpc_32_sse4;
+        c->lpc32 = ff_flac_lpc_32_sse4;
     }
     if (EXTERNAL_AVX(cpu_flags)) {
         if (fmt == AV_SAMPLE_FMT_S16) {
@@ -102,15 +101,14 @@ av_cold void ff_flacdsp_init_x86(FLACDSPContext *c, enum AVSampleFormat fmt, int
         }
     }
     if (EXTERNAL_XOP(cpu_flags)) {
-        if (bps > 16)
-            c->lpc = ff_flac_lpc_32_xop;
+        c->lpc32 = ff_flac_lpc_32_xop;
     }
 #endif
 
 #if CONFIG_FLAC_ENCODER
     if (EXTERNAL_SSE4(cpu_flags)) {
-        if (CONFIG_GPL && bps == 16)
-            c->lpc_encode = ff_flac_enc_lpc_16_sse4;
+        if (CONFIG_GPL)
+            c->lpc16_encode = ff_flac_enc_lpc_16_sse4;
     }
 #endif
 #endif /* HAVE_YASM */