libfdk-aacdec: Enable Decoder Downmix including Downmix Metadata Support

The FDK decoder is capable of producing mono and stereo downmix from
multichannel streams. These streams may contain metadata that control
the downmix process. The decoder requires an Ancillary Buffer in order to
correctly apply downmix in streams containing downmix Metadata. The
decoder does not have an API interface to inform of the presence of
Metadata in the stream, and therefore the Ancillary Buffer is always
allocated whenever a downmix is requested.

When downmixing multichannel streams, the decoder requires the output
buffer in aacDecoder_DecodeFrame call to be of fixed size in order to
hold the actual number of channels contained in the stream. For example,
for a 5.1ch to stereo downmix, the decoder requires that the output buffer
is allocated for 6 channels, regardless of the fact that the output is in
fact two channels.

Due to this requirement, the output buffer is allocated for the maximum
output buffer size in case a downmix is requested (and also during
decoder init). When a downmix is requested, the buffer used for output
during init will also be used for the entire duration the decoder is open.
Otherwise, the initial decoder output buffer is freed and the decoder
decodes straight into the output AVFrame.

Signed-off-by: Martin Storsjö <martin@martin.st>

1 files changed, 72 insertions, 8 deletions

diff --git a/libavcodec/libfdk-aacdec.c b/libavcodec/libfdk-aacdec.c
index 27e5712096..216e1c8f7c 100644
--- a/libavcodec/libfdk-aacdec.c
+++ b/libavcodec/libfdk-aacdec.c
@@ -36,9 +36,16 @@ typedef struct FDKAACDecContext {
     const AVClass *class;
     HANDLE_AACDECODER handle;
     int initialized;
+    uint8_t *decoder_buffer;
+    uint8_t *anc_buffer;
     enum ConcealMethod conceal_method;
 } FDKAACDecContext;
 
+
+#define DMX_ANC_BUFFSIZE       128
+#define DECODER_MAX_CHANNELS     6
+#define DECODER_BUFFSIZE      2048 * sizeof(INT_PCM)
+
 #define OFFSET(x) offsetof(FDKAACDecContext, x)
 #define AD AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_DECODING_PARAM
 static const AVOption fdk_aac_dec_options[] = {
@@ -170,6 +177,8 @@ static av_cold int fdk_aac_decode_close(AVCodecContext *avctx)
 
     if (s->handle)
         aacDecoder_Close(s->handle);
+    av_free(s->decoder_buffer);
+    av_free(s->anc_buffer);
 
     return 0;
 }
@@ -178,6 +187,7 @@ static av_cold int fdk_aac_decode_init(AVCodecContext *avctx)
 {
     FDKAACDecContext *s = avctx->priv_data;
     AAC_DECODER_ERROR err;
+    int ret;
 
     s->handle = aacDecoder_Open(avctx->extradata_size ? TT_MP4_RAW : TT_MP4_ADTS, 1);
     if (!s->handle) {
@@ -199,9 +209,49 @@ static av_cold int fdk_aac_decode_init(AVCodecContext *avctx)
         return AVERROR_UNKNOWN;
     }
 
+    if (avctx->request_channel_layout > 0 &&
+        avctx->request_channel_layout != AV_CH_LAYOUT_NATIVE) {
+        int downmix_channels = -1;
+
+        switch (avctx->request_channel_layout) {
+        case AV_CH_LAYOUT_STEREO:
+        case AV_CH_LAYOUT_STEREO_DOWNMIX:
+            downmix_channels = 2;
+            break;
+        case AV_CH_LAYOUT_MONO:
+            downmix_channels = 1;
+            break;
+        default:
+            av_log(avctx, AV_LOG_WARNING, "Invalid request_channel_layout\n");
+            break;
+        }
+
+        if (downmix_channels != -1) {
+            if (aacDecoder_SetParam(s->handle, AAC_PCM_OUTPUT_CHANNELS,
+                                    downmix_channels) != AAC_DEC_OK) {
+               av_log(avctx, AV_LOG_WARNING, "Unable to set output channels in the decoder\n");
+            } else {
+               s->anc_buffer = av_malloc(DMX_ANC_BUFFSIZE);
+               if (!s->anc_buffer) {
+                   av_log(avctx, AV_LOG_ERROR, "Unable to allocate ancillary buffer for the decoder\n");
+                   ret = AVERROR(ENOMEM);
+                   goto fail;
+               }
+               if (aacDecoder_AncDataInit(s->handle, s->anc_buffer, DMX_ANC_BUFFSIZE)) {
+                   av_log(avctx, AV_LOG_ERROR, "Unable to register downmix ancillary buffer in the decoder\n");
+                   ret = AVERROR_UNKNOWN;
+                   goto fail;
+               }
+            }
+        }
+    }
+
     avctx->sample_fmt = AV_SAMPLE_FMT_S16;
 
     return 0;
+fail:
+    fdk_aac_decode_close(avctx);
+    return ret;
 }
 
 static int fdk_aac_decode_frame(AVCodecContext *avctx, void *data,
@@ -227,14 +277,24 @@ static int fdk_aac_decode_frame(AVCodecContext *avctx, void *data,
             av_log(avctx, AV_LOG_ERROR, "ff_get_buffer() failed\n");
             return ret;
         }
-        buf = frame->extended_data[0];
-        buf_size = avctx->channels * frame->nb_samples *
-                   av_get_bytes_per_sample(avctx->sample_fmt);
+
+        if (s->anc_buffer) {
+            buf_size = DECODER_BUFFSIZE * DECODER_MAX_CHANNELS;
+            buf = s->decoder_buffer;
+        } else {
+            buf = frame->extended_data[0];
+            buf_size = avctx->channels * frame->nb_samples *
+                       av_get_bytes_per_sample(avctx->sample_fmt);
+        }
     } else {
-        buf_size = 50 * 1024;
-        buf = tmpptr = av_malloc(buf_size);
-        if (!buf)
+        buf_size = DECODER_BUFFSIZE * DECODER_MAX_CHANNELS;
+
+        if (!s->decoder_buffer)
+            s->decoder_buffer = av_malloc(buf_size);
+        if (!s->decoder_buffer)
             return AVERROR(ENOMEM);
+
+        buf = tmpptr = s->decoder_buffer;
     }
 
     err = aacDecoder_DecodeFrame(s->handle, (INT_PCM *) buf, buf_size, 0);
@@ -262,16 +322,20 @@ static int fdk_aac_decode_frame(AVCodecContext *avctx, void *data,
             av_log(avctx, AV_LOG_ERROR, "ff_get_buffer() failed\n");
             goto end;
         }
-        memcpy(frame->extended_data[0], tmpptr,
+    }
+    if (s->decoder_buffer) {
+        memcpy(frame->extended_data[0], buf,
                avctx->channels * avctx->frame_size *
                av_get_bytes_per_sample(avctx->sample_fmt));
+
+        if (!s->anc_buffer)
+            av_freep(&s->decoder_buffer);
     }
 
     *got_frame_ptr = 1;
     ret = avpkt->size - valid;
 
 end:
-    av_free(tmpptr);
     return ret;
 }