avcodec/wmalosslessdec: ensure channel count in the private context and decoder context are consistent

Fixes: Out of array write
Fixes: 45613/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_WMALOSSLESS_fuzzer-4539073606320128
Fixes: 46008/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_WMALOSSLESS_fuzzer-4681245747970048

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg

Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Signed-off-by: James Almer <jamrial@gmail.com>

1 files changed, 26 insertions, 23 deletions

diff --git a/libavcodec/wmalosslessdec.c b/libavcodec/wmalosslessdec.c
index 7d5753dccd..d4f3462804 100644
--- a/libavcodec/wmalosslessdec.c
+++ b/libavcodec/wmalosslessdec.c
@@ -198,15 +198,6 @@ static av_cold int decode_init(AVCodecContext *avctx)
         return AVERROR_PATCHWELCOME;
     }
 
-    s->max_frame_size = MAX_FRAMESIZE * avctx->ch_layout.nb_channels;
-    s->frame_data = av_mallocz(s->max_frame_size + AV_INPUT_BUFFER_PADDING_SIZE);
-    if (!s->frame_data)
-        return AVERROR(ENOMEM);
-
-    s->avctx = avctx;
-    ff_llauddsp_init(&s->dsp);
-    init_put_bits(&s->pb, s->frame_data, s->max_frame_size);
-
     if (avctx->extradata_size >= 18) {
         s->decode_flags    = AV_RL16(edata_ptr + 14);
         channel_mask       = AV_RL32(edata_ptr +  2);
@@ -231,6 +222,32 @@ static av_cold int decode_init(AVCodecContext *avctx)
         return AVERROR_PATCHWELCOME;
     }
 
+    if (channel_mask) {
+        av_channel_layout_uninit(&avctx->ch_layout);
+        av_channel_layout_from_mask(&avctx->ch_layout, channel_mask);
+    }
+
+    s->num_channels = avctx->ch_layout.nb_channels;
+
+    /* extract lfe channel position */
+    s->lfe_channel = -1;
+
+    if (channel_mask & 8) {
+        unsigned int mask;
+        for (mask = 1; mask < 16; mask <<= 1)
+            if (channel_mask & mask)
+                ++s->lfe_channel;
+    }
+
+    s->max_frame_size = MAX_FRAMESIZE * avctx->ch_layout.nb_channels;
+    s->frame_data = av_mallocz(s->max_frame_size + AV_INPUT_BUFFER_PADDING_SIZE);
+    if (!s->frame_data)
+        return AVERROR(ENOMEM);
+
+    s->avctx = avctx;
+    ff_llauddsp_init(&s->dsp);
+    init_put_bits(&s->pb, s->frame_data, s->max_frame_size);
+
     /* generic init */
     s->log2_frame_size = av_log2(avctx->block_align) + 4;
 
@@ -264,24 +281,10 @@ static av_cold int decode_init(AVCodecContext *avctx)
         return AVERROR_INVALIDDATA;
     }
 
-    s->num_channels = avctx->ch_layout.nb_channels;
-
-    /* extract lfe channel position */
-    s->lfe_channel = -1;
-
-    if (channel_mask & 8) {
-        unsigned int mask;
-        for (mask = 1; mask < 16; mask <<= 1)
-            if (channel_mask & mask)
-                ++s->lfe_channel;
-    }
-
     s->frame = av_frame_alloc();
     if (!s->frame)
         return AVERROR(ENOMEM);
 
-    av_channel_layout_uninit(&avctx->ch_layout);
-    av_channel_layout_from_mask(&avctx->ch_layout, channel_mask);
     return 0;
 }