audiotoolbox: convert to new channel layout API

Signed-off-by: James Almer <jamrial@gmail.com>

2 files changed, 61 insertions, 64 deletions

diff --git a/libavcodec/audiotoolboxdec.c b/libavcodec/audiotoolboxdec.c
index 9939fef218..6f9c2ab746 100644
--- a/libavcodec/audiotoolboxdec.c
+++ b/libavcodec/audiotoolboxdec.c
@@ -166,8 +166,8 @@ static int ffat_update_ctx(AVCodecContext *avctx)
                                    &size, &format)) {
         if (format.mSampleRate)
             avctx->sample_rate = format.mSampleRate;
-        avctx->channels = format.mChannelsPerFrame;
-        avctx->channel_layout = av_get_default_channel_layout(avctx->channels);
+        av_channel_layout_uninit(&avctx->ch_layout);
+        av_channel_layout_default(&avctx->ch_layout, format.mChannelsPerFrame);
         avctx->frame_size = format.mFramesPerPacket;
     }
 
@@ -175,7 +175,7 @@ static int ffat_update_ctx(AVCodecContext *avctx)
                                    kAudioConverterCurrentOutputStreamDescription,
                                    &size, &format)) {
         format.mSampleRate = avctx->sample_rate;
-        format.mChannelsPerFrame = avctx->channels;
+        format.mChannelsPerFrame = avctx->ch_layout.nb_channels;
         AudioConverterSetProperty(at->converter,
                                   kAudioConverterCurrentOutputStreamDescription,
                                   size, &format);
@@ -201,7 +201,8 @@ static int ffat_update_ctx(AVCodecContext *avctx)
             layout_mask |= 1 << id;
             layout->mChannelDescriptions[i].mChannelFlags = i; // Abusing flags as index
         }
-        avctx->channel_layout = layout_mask;
+        av_channel_layout_uninit(&avctx->ch_layout);
+        av_channel_layout_from_mask(&avctx->ch_layout, layout_mask);
         qsort(layout->mChannelDescriptions, layout->mNumberChannelDescriptions,
               sizeof(AudioChannelDescription), &ffat_compare_channel_descriptions);
         for (i = 0; i < layout->mNumberChannelDescriptions; i++)
@@ -364,11 +365,13 @@ static av_cold int ffat_create_decoder(AVCodecContext *avctx,
 #endif
     } else {
         in_format.mSampleRate = avctx->sample_rate ? avctx->sample_rate : 44100;
-        in_format.mChannelsPerFrame = avctx->channels ? avctx->channels : 1;
+        in_format.mChannelsPerFrame = avctx->ch_layout.nb_channels ? avctx->ch_layout.nb_channels : 1;
     }
 
     avctx->sample_rate = out_format.mSampleRate = in_format.mSampleRate;
-    avctx->channels = out_format.mChannelsPerFrame = in_format.mChannelsPerFrame;
+    av_channel_layout_uninit(&avctx->ch_layout);
+    avctx->ch_layout.order       = AV_CHANNEL_ORDER_UNSPEC;
+    avctx->ch_layout.nb_channels = out_format.mChannelsPerFrame = in_format.mChannelsPerFrame;
 
     if (avctx->codec_id == AV_CODEC_ID_ADPCM_IMA_QT)
         in_format.mFramesPerPacket = 64;
@@ -389,7 +392,7 @@ static av_cold int ffat_create_decoder(AVCodecContext *avctx,
     ffat_update_ctx(avctx);
 
     if(!(at->decoded_data = av_malloc(av_get_bytes_per_sample(avctx->sample_fmt)
-                                      * avctx->frame_size * avctx->channels)))
+                                      * avctx->frame_size * avctx->ch_layout.nb_channels)))
         return AVERROR(ENOMEM);
 
     at->last_pts = AV_NOPTS_VALUE;
@@ -408,7 +411,7 @@ static av_cold int ffat_init_decoder(AVCodecContext *avctx)
         memcpy(at->extradata, avctx->extradata, avctx->extradata_size);
     }
 
-    if ((avctx->channels && avctx->sample_rate) || ffat_usable_extradata(avctx))
+    if ((avctx->ch_layout.nb_channels && avctx->sample_rate) || ffat_usable_extradata(avctx))
         return ffat_create_decoder(avctx, NULL);
     else
         return 0;
@@ -455,11 +458,11 @@ static OSStatus ffat_decode_callback(AudioConverterRef converter, UInt32 *nb_pac
 
 #define COPY_SAMPLES(type) \
     type *in_ptr = (type*)at->decoded_data; \
-    type *end_ptr = in_ptr + frame->nb_samples * avctx->channels; \
+    type *end_ptr = in_ptr + frame->nb_samples * avctx->ch_layout.nb_channels; \
     type *out_ptr = (type*)frame->data[0]; \
-    for (; in_ptr < end_ptr; in_ptr += avctx->channels, out_ptr += avctx->channels) { \
+    for (; in_ptr < end_ptr; in_ptr += avctx->ch_layout.nb_channels, out_ptr += avctx->ch_layout.nb_channels) { \
         int c; \
-        for (c = 0; c < avctx->channels; c++) \
+        for (c = 0; c < avctx->ch_layout.nb_channels; c++) \
             out_ptr[c] = in_ptr[at->channel_map[c]]; \
     }
 
@@ -509,9 +512,9 @@ static int ffat_decode(AVCodecContext *avctx, void *data,
         .mNumberBuffers = 1,
         .mBuffers = {
             {
-                .mNumberChannels = avctx->channels,
+                .mNumberChannels = avctx->ch_layout.nb_channels,
                 .mDataByteSize = av_get_bytes_per_sample(avctx->sample_fmt) * avctx->frame_size
-                                 * avctx->channels,
+                                 * avctx->ch_layout.nb_channels,
             }
         }
     };
diff --git a/libavcodec/audiotoolboxenc.c b/libavcodec/audiotoolboxenc.c
index 9245aa9dc4..ed7985b06a 100644
--- a/libavcodec/audiotoolboxenc.c
+++ b/libavcodec/audiotoolboxenc.c
@@ -178,18 +178,17 @@ static av_cold int get_channel_label(int channel)
         return -1;
 }
 
-static int remap_layout(AudioChannelLayout *layout, uint64_t in_layout, int count)
+static int remap_layout(AudioChannelLayout *layout, const AVChannelLayout *in_layout)
 {
     int i;
-    int c = 0;
     layout->mChannelLayoutTag = kAudioChannelLayoutTag_UseChannelDescriptions;
-    layout->mNumberChannelDescriptions = count;
-    for (i = 0; i < count; i++) {
-        int label;
-        while (!(in_layout & (1 << c)) && c < 64)
-            c++;
-        if (c == 64)
-            return AVERROR(EINVAL); // This should never happen
+    layout->mNumberChannelDescriptions = in_layout->nb_channels;
+    for (i = 0; i < in_layout->nb_channels; i++) {
+        int c, label;
+
+        c = av_channel_layout_get_channel(in_layout, i);
+        if (c < 0 || c >= 64)
+            return AVERROR(EINVAL);
         label = get_channel_label(c);
         layout->mChannelDescriptions[i].mChannelLabel = label;
         if (label < 0)
@@ -199,38 +198,33 @@ static int remap_layout(AudioChannelLayout *layout, uint64_t in_layout, int coun
     return 0;
 }
 
-static int get_aac_tag(uint64_t in_layout)
+static int get_aac_tag(const AVChannelLayout *in_layout)
 {
-    switch (in_layout) {
-    case AV_CH_LAYOUT_MONO:
-        return kAudioChannelLayoutTag_Mono;
-    case AV_CH_LAYOUT_STEREO:
-        return kAudioChannelLayoutTag_Stereo;
-    case AV_CH_LAYOUT_QUAD:
-        return kAudioChannelLayoutTag_AAC_Quadraphonic;
-    case AV_CH_LAYOUT_OCTAGONAL:
-        return kAudioChannelLayoutTag_AAC_Octagonal;
-    case AV_CH_LAYOUT_SURROUND:
-        return kAudioChannelLayoutTag_AAC_3_0;
-    case AV_CH_LAYOUT_4POINT0:
-        return kAudioChannelLayoutTag_AAC_4_0;
-    case AV_CH_LAYOUT_5POINT0:
-        return kAudioChannelLayoutTag_AAC_5_0;
-    case AV_CH_LAYOUT_5POINT1:
-        return kAudioChannelLayoutTag_AAC_5_1;
-    case AV_CH_LAYOUT_6POINT0:
-        return kAudioChannelLayoutTag_AAC_6_0;
-    case AV_CH_LAYOUT_6POINT1:
-        return kAudioChannelLayoutTag_AAC_6_1;
-    case AV_CH_LAYOUT_7POINT0:
-        return kAudioChannelLayoutTag_AAC_7_0;
-    case AV_CH_LAYOUT_7POINT1_WIDE_BACK:
-        return kAudioChannelLayoutTag_AAC_7_1;
-    case AV_CH_LAYOUT_7POINT1:
-        return kAudioChannelLayoutTag_MPEG_7_1_C;
-    default:
-        return 0;
-    }
+    static const struct {
+        AVChannelLayout chl;
+        int tag;
+    } map[] = {
+        { AV_CH_LAYOUT_MONO,              kAudioChannelLayoutTag_Mono },
+        { AV_CH_LAYOUT_STEREO,            kAudioChannelLayoutTag_Stereo },
+        { AV_CH_LAYOUT_QUAD,              kAudioChannelLayoutTag_AAC_Quadraphonic },
+        { AV_CH_LAYOUT_OCTAGONAL,         kAudioChannelLayoutTag_AAC_Octagonal },
+        { AV_CH_LAYOUT_SURROUND,          kAudioChannelLayoutTag_AAC_3_0 },
+        { AV_CH_LAYOUT_4POINT0,           kAudioChannelLayoutTag_AAC_4_0 },
+        { AV_CH_LAYOUT_5POINT0,           kAudioChannelLayoutTag_AAC_5_0 },
+        { AV_CH_LAYOUT_5POINT1,           kAudioChannelLayoutTag_AAC_5_1 },
+        { AV_CH_LAYOUT_6POINT0,           kAudioChannelLayoutTag_AAC_6_0 },
+        { AV_CH_LAYOUT_6POINT1,           kAudioChannelLayoutTag_AAC_6_1 },
+        { AV_CH_LAYOUT_7POINT0,           kAudioChannelLayoutTag_AAC_7_0 },
+        { AV_CH_LAYOUT_7POINT1_WIDE_BACK, kAudioChannelLayoutTag_AAC_7_1 },
+        { AV_CH_LAYOUT_7POINT1,           kAudioChannelLayoutTag_MPEG_7_1_C },
+    };
+    int i;
+
+    for (i = 0; i < FF_ARRAY_ELEMS; i++)
+        if (!av_channel_layout_compare(in_layout, &map[i].chl))
+            return map[i].tag;
+
+    return 0;
 }
 
 static av_cold int ffat_init_encoder(AVCodecContext *avctx)
@@ -246,10 +240,10 @@ static av_cold int ffat_init_encoder(AVCodecContext *avctx)
                         : avctx->sample_fmt == AV_SAMPLE_FMT_U8 ? 0
                         : kAudioFormatFlagIsSignedInteger)
                         | kAudioFormatFlagIsPacked,
-        .mBytesPerPacket = av_get_bytes_per_sample(avctx->sample_fmt) * avctx->channels,
+        .mBytesPerPacket = av_get_bytes_per_sample(avctx->sample_fmt) * avctx->ch_layout.nb_channels,
         .mFramesPerPacket = 1,
-        .mBytesPerFrame = av_get_bytes_per_sample(avctx->sample_fmt) * avctx->channels,
-        .mChannelsPerFrame = avctx->channels,
+        .mBytesPerFrame = av_get_bytes_per_sample(avctx->sample_fmt) * avctx->ch_layout.nb_channels,
+        .mChannelsPerFrame = avctx->ch_layout.nb_channels,
         .mBitsPerChannel = av_get_bytes_per_sample(avctx->sample_fmt) * 8,
     };
     AudioStreamBasicDescription out_format = {
@@ -258,7 +252,7 @@ static av_cold int ffat_init_encoder(AVCodecContext *avctx)
         .mChannelsPerFrame = in_format.mChannelsPerFrame,
     };
     UInt32 layout_size = sizeof(AudioChannelLayout) +
-                         sizeof(AudioChannelDescription) * avctx->channels;
+                         sizeof(AudioChannelDescription) * avctx->ch_layout.nb_channels;
     AudioChannelLayout *channel_layout = av_malloc(layout_size);
 
     if (!channel_layout)
@@ -278,10 +272,10 @@ static av_cold int ffat_init_encoder(AVCodecContext *avctx)
         return AVERROR_UNKNOWN;
     }
 
-    if (!avctx->channel_layout)
-        avctx->channel_layout = av_get_default_channel_layout(avctx->channels);
+    if (avctx->ch_layout.order == AV_CHANNEL_ORDER_UNSPEC)
+        av_channel_layout_default(&avctx->ch_layout, avctx->ch_layout.nb_channels);
 
-    if ((status = remap_layout(channel_layout, avctx->channel_layout, avctx->channels)) < 0) {
+    if ((status = remap_layout(channel_layout, &avctx->ch_layout)) < 0) {
         av_log(avctx, AV_LOG_ERROR, "Invalid channel layout\n");
         av_free(channel_layout);
         return status;
@@ -294,7 +288,7 @@ static av_cold int ffat_init_encoder(AVCodecContext *avctx)
         return AVERROR(EINVAL);
     }
     if (avctx->codec_id == AV_CODEC_ID_AAC) {
-        int tag = get_aac_tag(avctx->channel_layout);
+        int tag = get_aac_tag(&avctx->ch_layout);
         if (tag) {
             channel_layout->mChannelLayoutTag = tag;
             channel_layout->mNumberChannelDescriptions = 0;
@@ -476,10 +470,10 @@ static OSStatus ffat_encode_callback(AudioConverterRef converter, UInt32 *nb_pac
     frame = ff_bufqueue_get(&at->frame_queue);
 
     data->mNumberBuffers              = 1;
-    data->mBuffers[0].mNumberChannels = avctx->channels;
+    data->mBuffers[0].mNumberChannels = avctx->ch_layout.nb_channels;
     data->mBuffers[0].mDataByteSize   = frame->nb_samples *
                                         av_get_bytes_per_sample(avctx->sample_fmt) *
-                                        avctx->channels;
+                                        avctx->ch_layout.nb_channels;
     data->mBuffers[0].mData           = frame->data[0];
     if (*nb_packets > frame->nb_samples)
         *nb_packets = frame->nb_samples;
@@ -506,7 +500,7 @@ static int ffat_encode(AVCodecContext *avctx, AVPacket *avpkt,
         .mNumberBuffers = 1,
         .mBuffers = {
             {
-                .mNumberChannels = avctx->channels,
+                .mNumberChannels = avctx->ch_layout.nb_channels,
                 .mDataByteSize = at->pkt_size,
             }
         }