libfdk-aac: convert to new channel layout API

Signed-off-by: Anton Khirnov <anton@khirnov.net>
Signed-off-by: James Almer <jamrial@gmail.com>

2 files changed, 48 insertions, 21 deletions

diff --git a/libavcodec/libfdk-aacdec.c b/libavcodec/libfdk-aacdec.c
index d560e313ca..d7b84015be 100644
--- a/libavcodec/libfdk-aacdec.c
+++ b/libavcodec/libfdk-aacdec.c
@@ -63,6 +63,7 @@ typedef struct FDKAACDecContext {
     int flush_samples;
     int delay_samples;
 #endif
+    AVChannelLayout downmix_layout;
 } FDKAACDecContext;
 
 
@@ -97,6 +98,7 @@ static const AVOption fdk_aac_dec_options[] = {
     { "album_mode","Dynamic Range Control: album mode, where [0] is off and [1] is on",
                      OFFSET(album_mode),     AV_OPT_TYPE_INT,   { .i64 = -1},  -1, 1,   AD, NULL    },
 #endif
+    { "downmix", "Request a specific channel layout from the decoder", OFFSET(downmix_layout), AV_OPT_TYPE_CHLAYOUT, {.str = NULL}, .flags = AD },
     { NULL }
 };
 
@@ -211,17 +213,15 @@ static int get_stream_info(AVCodecContext *avctx)
             ch_error = 1;
         }
     }
-    if (!ch_error &&
-        av_get_channel_layout_nb_channels(ch_layout) != info->numChannels) {
+
+    av_channel_layout_uninit(&avctx->ch_layout);
+    av_channel_layout_from_mask(&avctx->ch_layout, ch_layout);
+    if (!ch_error && avctx->ch_layout.nb_channels != info->numChannels) {
         av_log(avctx, AV_LOG_WARNING, "unsupported channel configuration\n");
         ch_error = 1;
     }
     if (ch_error)
-        avctx->channel_layout = 0;
-    else
-        avctx->channel_layout = ch_layout;
-
-    avctx->channels = info->numChannels;
+        avctx->ch_layout.order = AV_CHANNEL_ORDER_UNSPEC;
 
     return 0;
 }
@@ -263,11 +263,19 @@ static av_cold int fdk_aac_decode_init(AVCodecContext *avctx)
         return AVERROR_UNKNOWN;
     }
 
-    if (avctx->request_channel_layout > 0 &&
-        avctx->request_channel_layout != AV_CH_LAYOUT_NATIVE) {
+#if FF_API_OLD_CHANNEL_LAYOUT
+FF_DISABLE_DEPRECATION_WARNINGS
+    if (avctx->request_channel_layout) {
+        av_channel_layout_uninit(&s->downmix_layout);
+        av_channel_layout_from_mask(&s->downmix_layout, avctx->request_channel_layout);
+    }
+FF_ENABLE_DEPRECATION_WARNINGS
+#endif
+    if (s->downmix_layout.nb_channels > 0 &&
+        s->downmix_layout.order != AV_CHANNEL_ORDER_NATIVE) {
         int downmix_channels = -1;
 
-        switch (avctx->request_channel_layout) {
+        switch (s->downmix_layout.u.mask) {
         case AV_CH_LAYOUT_STEREO:
         case AV_CH_LAYOUT_STEREO_DOWNMIX:
             downmix_channels = 2;
@@ -276,7 +284,7 @@ static av_cold int fdk_aac_decode_init(AVCodecContext *avctx)
             downmix_channels = 1;
             break;
         default:
-            av_log(avctx, AV_LOG_WARNING, "Invalid request_channel_layout\n");
+            av_log(avctx, AV_LOG_WARNING, "Invalid downmix option\n");
             break;
         }
 
@@ -434,7 +442,7 @@ static int fdk_aac_decode_frame(AVCodecContext *avctx, void *data,
                                     drop_samples, s->delay_samples);
             s->delay_samples  -= drop_samples;
             frame->nb_samples -= drop_samples;
-            input_offset = drop_samples * avctx->channels;
+            input_offset = drop_samples * avctx->ch_layout.nb_channels;
             if (frame->nb_samples <= 0)
                 return 0;
         }
@@ -445,7 +453,7 @@ static int fdk_aac_decode_frame(AVCodecContext *avctx, void *data,
         goto end;
 
     memcpy(frame->extended_data[0], s->decoder_buffer + input_offset,
-           avctx->channels * frame->nb_samples *
+           avctx->ch_layout.nb_channels * frame->nb_samples *
            av_get_bytes_per_sample(avctx->sample_fmt));
 
     *got_frame_ptr = 1;
diff --git a/libavcodec/libfdk-aacenc.c b/libavcodec/libfdk-aacenc.c
index 7ee2f13ac7..936049f12b 100644
--- a/libavcodec/libfdk-aacenc.c
+++ b/libavcodec/libfdk-aacenc.c
@@ -128,7 +128,7 @@ static av_cold int aac_encode_init(AVCodecContext *avctx)
     int aot = FF_PROFILE_AAC_LOW + 1;
     int sce = 0, cpe = 0;
 
-    if ((err = aacEncOpen(&s->handle, 0, avctx->channels)) != AACENC_OK) {
+    if ((err = aacEncOpen(&s->handle, 0, avctx->ch_layout.nb_channels)) != AACENC_OK) {
         av_log(avctx, AV_LOG_ERROR, "Unable to open the encoder: %s\n",
                aac_get_error(err));
         goto error;
@@ -159,7 +159,7 @@ static av_cold int aac_encode_init(AVCodecContext *avctx)
         goto error;
     }
 
-    switch (avctx->channels) {
+    switch (avctx->ch_layout.nb_channels) {
     case 1: mode = MODE_1;       sce = 1; cpe = 0; break;
     case 2:
 #if FDKENC_VER_AT_LEAST(4, 0) // 4.0.0
@@ -193,7 +193,7 @@ static av_cold int aac_encode_init(AVCodecContext *avctx)
     case 8:
         sce = 2;
         cpe = 3;
-        if (avctx->channel_layout == AV_CH_LAYOUT_7POINT1) {
+        if (!av_channel_layout_compare(&avctx->ch_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_7POINT1)) {
             mode = MODE_7_1_REAR_SURROUND;
         } else {
             // MODE_1_2_2_2_1 and MODE_7_1_FRONT_CENTER use the same channel layout
@@ -203,7 +203,7 @@ static av_cold int aac_encode_init(AVCodecContext *avctx)
 #endif
     default:
         av_log(avctx, AV_LOG_ERROR,
-               "Unsupported number of channels %d\n", avctx->channels);
+               "Unsupported number of channels %d\n", avctx->ch_layout.nb_channels);
         goto error;
     }
 
@@ -375,9 +375,9 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
         in_args.numInSamples = -1;
     } else {
         in_ptr               = frame->data[0];
-        in_buffer_size       = 2 * avctx->channels * frame->nb_samples;
+        in_buffer_size       = 2 * avctx->ch_layout.nb_channels * frame->nb_samples;
 
-        in_args.numInSamples = avctx->channels * frame->nb_samples;
+        in_args.numInSamples = avctx->ch_layout.nb_channels * frame->nb_samples;
 
         /* add current frame to the queue */
         if ((ret = ff_af_queue_add(&s->afq, frame)) < 0)
@@ -392,7 +392,7 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
     in_buf.bufElSizes        = &in_buffer_element_size;
 
     /* The maximum packet size is 6144 bits aka 768 bytes per channel. */
-    ret = ff_alloc_packet(avctx, avpkt, FFMAX(8192, 768 * avctx->channels));
+    ret = ff_alloc_packet(avctx, avpkt, FFMAX(8192, 768 * avctx->ch_layout.nb_channels));
     if (ret < 0)
         return ret;
 
@@ -440,6 +440,7 @@ static const AVCodecDefault aac_encode_defaults[] = {
     { NULL }
 };
 
+#if FF_API_OLD_CHANNEL_LAYOUT
 static const uint64_t aac_channel_layout[] = {
     AV_CH_LAYOUT_MONO,
     AV_CH_LAYOUT_STEREO,
@@ -453,6 +454,21 @@ static const uint64_t aac_channel_layout[] = {
 #endif
     0,
 };
+#endif /* FF_API_OLD_CHANNEL_LAYOUT */
+
+static const AVChannelLayout aac_ch_layouts[16] = {
+    AV_CHANNEL_LAYOUT_MONO,
+    AV_CHANNEL_LAYOUT_STEREO,
+    AV_CHANNEL_LAYOUT_SURROUND,
+    AV_CHANNEL_LAYOUT_4POINT0,
+    AV_CHANNEL_LAYOUT_5POINT0_BACK,
+    AV_CHANNEL_LAYOUT_5POINT1_BACK,
+#ifdef AACENCODER_LIB_VL0
+    AV_CHANNEL_LAYOUT_7POINT1_WIDE_BACK,
+    AV_CHANNEL_LAYOUT_7POINT1,
+#endif
+    { 0 },
+};
 
 static const int aac_sample_rates[] = {
     96000, 88200, 64000, 48000, 44100, 32000,
@@ -475,6 +491,9 @@ const AVCodec ff_libfdk_aac_encoder = {
     .defaults              = aac_encode_defaults,
     .profiles              = profiles,
     .supported_samplerates = aac_sample_rates,
-    .channel_layouts       = aac_channel_layout,
     .wrapper_name          = "libfdk",
+#if FF_API_OLD_CHANNEL_LAYOUT
+    .channel_layouts       = aac_channel_layout,
+#endif
+    .ch_layouts            = aac_ch_layouts,
 };