msmpeg4: check memory allocations and propagate errors

Bug-Id: CID 1257781

1 files changed, 14 insertions, 5 deletions

diff --git a/libavcodec/msmpeg4enc.c b/libavcodec/msmpeg4enc.c
index 45ef208935..999c541b58 100644
--- a/libavcodec/msmpeg4enc.c
+++ b/libavcodec/msmpeg4enc.c
@@ -46,11 +46,14 @@
 static uint8_t rl_length[NB_RL_TABLES][MAX_LEVEL+1][MAX_RUN+1][2];
 
 /* build the table which associate a (x,y) motion vector to a vlc */
-static av_cold void init_mv_table(MVTable *tab)
+static av_cold int init_mv_table(MVTable *tab)
 {
     int i, x, y;
 
     tab->table_mv_index = av_malloc(sizeof(uint16_t) * 4096);
+    if (!tab->table_mv_index)
+        return AVERROR(ENOMEM);
+
     /* mark all entries as not used */
     for(i=0;i<4096;i++)
         tab->table_mv_index[i] = tab->n;
@@ -60,6 +63,8 @@ static av_cold void init_mv_table(MVTable *tab)
         y = tab->table_mvy[i];
         tab->table_mv_index[(x << 6) | y] = i;
     }
+
+    return 0;
 }
 
 void ff_msmpeg4_code012(PutBitContext *pb, int n)
@@ -113,10 +118,10 @@ static int get_size_of_code(MpegEncContext * s, RLTable *rl, int last, int run,
     return size;
 }
 
-av_cold void ff_msmpeg4_encode_init(MpegEncContext *s)
+av_cold int ff_msmpeg4_encode_init(MpegEncContext *s)
 {
     static int init_done=0;
-    int i;
+    int i, ret;
 
     ff_msmpeg4_common_init(s);
     if(s->msmpeg4_version>=4){
@@ -127,8 +132,10 @@ av_cold void ff_msmpeg4_encode_init(MpegEncContext *s)
     if (!init_done) {
         /* init various encoding tables */
         init_done = 1;
-        init_mv_table(&ff_mv_tables[0]);
-        init_mv_table(&ff_mv_tables[1]);
+        if ((ret = init_mv_table(&ff_mv_tables[0])) < 0)
+            return ret;
+        if ((ret = init_mv_table(&ff_mv_tables[1])) < 0)
+            return ret;
         for(i=0;i<NB_RL_TABLES;i++)
             ff_init_rl(&ff_rl_table[i], ff_static_rl_table_store[i]);
 
@@ -145,6 +152,8 @@ av_cold void ff_msmpeg4_encode_init(MpegEncContext *s)
             }
         }
     }
+
+    return 0;
 }
 
 static void find_best_tables(MpegEncContext * s)