cook: check that category is smaller than 8

This fixes some out of global array accesses of dither_tab.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Reviewed-by: Benjamin Larsson <benjamin@southpole.se>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 5 insertions, 1 deletions

diff --git a/libavcodec/cook.c b/libavcodec/cook.c
index 6c111de785..294044e7f2 100644
--- a/libavcodec/cook.c
+++ b/libavcodec/cook.c
@@ -647,7 +647,7 @@ static int mono_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer)
     int category_index[128];
     int quant_index_table[102];
     int category[128];
-    int ret;
+    int ret, i;
 
     memset(&category,       0, sizeof(category));
     memset(&category_index, 0, sizeof(category_index));
@@ -657,6 +657,10 @@ static int mono_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer)
     q->num_vectors = get_bits(&q->gb, p->log2_numvector_size);
     categorize(q, p, quant_index_table, category, category_index);
     expand_category(q, category, category_index);
+    for (i=0; i<p->total_subbands; i++) {
+        if (category[i] > 7)
+            return AVERROR_INVALIDDATA;
+    }
     decode_vectors(q, p, category, quant_index_table, mlt_buffer);
 
     return 0;