integer overflows, heap corruption

possible arbitrary code execution cannot be ruled out in some cases
precautionary checks

Originally committed as revision 3813 to svn://svn.ffmpeg.org/ffmpeg/trunk

19 files changed, 108 insertions, 26 deletions

diff --git a/libavformat/4xm.c b/libavformat/4xm.c
index 462917893e..39e1e87061 100644
--- a/libavformat/4xm.c
+++ b/libavformat/4xm.c
@@ -185,6 +185,8 @@ static int fourxm_read_header(AVFormatContext *s,
             current_track = LE_32(&header[i + 8]);
             if (current_track + 1 > fourxm->track_count) {
                 fourxm->track_count = current_track + 1;
+                if((unsigned)fourxm->track_count >= UINT_MAX / sizeof(AudioTrack))
+                    return -1;
                 fourxm->tracks = av_realloc(fourxm->tracks, 
                     fourxm->track_count * sizeof(AudioTrack));
                 if (!fourxm->tracks) {
diff --git a/libavformat/allformats.c b/libavformat/allformats.c
index b69efa316b..45b98fcd64 100644
--- a/libavformat/allformats.c
+++ b/libavformat/allformats.c
@@ -125,8 +125,8 @@ void av_register_all(void)
 #endif
     av_register_image_format(&jpeg_image_format);
 #endif
-    av_register_image_format(&gif_image_format);
-    av_register_image_format(&sgi_image_format);
+    av_register_image_format(&gif_image_format);  
+//    av_register_image_format(&sgi_image_format); heap corruption, dont enable
 #endif //CONFIG_ENCODERS
 
     /* file protocols */
diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index c04f0f33a1..67a2166f8c 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -546,7 +546,7 @@ int fifo_size(FifoBuffer *f, uint8_t *rptr);
 int fifo_read(FifoBuffer *f, uint8_t *buf, int buf_size, uint8_t **rptr_ptr);
 void fifo_write(FifoBuffer *f, uint8_t *buf, int size, uint8_t **wptr_ptr);
 int put_fifo(ByteIOContext *pb, FifoBuffer *f, int buf_size, uint8_t **rptr_ptr);
-void fifo_realloc(FifoBuffer *f, int size);
+void fifo_realloc(FifoBuffer *f, unsigned int size);
 
 /* media file input */
 AVInputFormat *av_find_input_format(const char *short_name);
diff --git a/libavformat/avidec.c b/libavformat/avidec.c
index fa33101106..060d3b926d 100644
--- a/libavformat/avidec.c
+++ b/libavformat/avidec.c
@@ -302,9 +302,11 @@ static int avi_read_header(AVFormatContext *s, AVFormatParameters *ap)
                     get_le32(pb); /* ClrUsed */
                     get_le32(pb); /* ClrImportant */
 
+                 if(size > 10*4 && size<(1<<30)){
                     st->codec.extradata_size= size - 10*4;
                     st->codec.extradata= av_malloc(st->codec.extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
                     get_buffer(pb, st->codec.extradata, st->codec.extradata_size);
+                 }
                     
                     if(st->codec.extradata_size & 1) //FIXME check if the encoder really did this correctly
                         get_byte(pb);
@@ -549,6 +551,8 @@ static int avi_read_idx1(AVFormatContext *s, int size)
     nb_index_entries = size / 16;
     if (nb_index_entries <= 0)
         return -1;
+    if(nb_index_entries + 1 >= UINT_MAX / sizeof(AVIIndexEntry))
+        return -1;
 
     /* read the entries and sort them in each stream component */
     for(i = 0; i < nb_index_entries; i++) {
diff --git a/libavformat/aviobuf.c b/libavformat/aviobuf.c
index 31c6a7fec5..bb55254532 100644
--- a/libavformat/aviobuf.c
+++ b/libavformat/aviobuf.c
@@ -629,11 +629,13 @@ static int dyn_buf_write(void *opaque, uint8_t *buf, int buf_size)
     /* reallocate buffer if needed */
     new_size = d->pos + buf_size;
     new_allocated_size = d->allocated_size;
+    if(new_size < d->pos || new_size > INT_MAX/2)
+        return -1;
     while (new_size > new_allocated_size) {
         if (!new_allocated_size)
             new_allocated_size = new_size;
         else
-            new_allocated_size = (new_allocated_size * 3) / 2 + 1;    
+            new_allocated_size += new_allocated_size / 2 + 1;    
     }
     
     if (new_allocated_size > d->allocated_size) {
@@ -691,6 +693,8 @@ static int url_open_dyn_buf_internal(ByteIOContext *s, int max_packet_size)
     else
         io_buffer_size = 1024;
         
+    if(sizeof(DynBuffer) + io_buffer_size < io_buffer_size)
+        return -1;
     d = av_malloc(sizeof(DynBuffer) + io_buffer_size);
     if (!d)
         return -1;
diff --git a/libavformat/gifdec.c b/libavformat/gifdec.c
index 133ce1718e..4ee295de7b 100644
--- a/libavformat/gifdec.c
+++ b/libavformat/gifdec.c
@@ -474,6 +474,12 @@ static int gif_read_header1(GifState *s)
     s->transparent_color_index = -1;
     s->screen_width = get_le16(f);
     s->screen_height = get_le16(f);
+    if(   (unsigned)s->screen_width  > 32767 
+       || (unsigned)s->screen_height > 32767){
+        av_log(NULL, AV_LOG_ERROR, "picture size too large\n");
+        return -1;
+    } 
+
     v = get_byte(f);
     s->color_resolution = ((v & 0x70) >> 4) + 1;
     has_global_palette = (v & 0x80);
diff --git a/libavformat/grab.c b/libavformat/grab.c
index ded571b377..84284c132c 100644
--- a/libavformat/grab.c
+++ b/libavformat/grab.c
@@ -76,6 +76,9 @@ static int grab_read_header(AVFormatContext *s1, AVFormatParameters *ap)
     frame_rate      = ap->frame_rate;
     frame_rate_base = ap->frame_rate_base;
 
+    if((unsigned)width > 32767 || (unsigned)height > 32767)
+        return -1;
+    
     st = av_new_stream(s1, 0);
     if (!st)
         return -ENOMEM;
diff --git a/libavformat/http.c b/libavformat/http.c
index d8ab4d3f44..85b1f319bd 100644
--- a/libavformat/http.c
+++ b/libavformat/http.c
@@ -290,12 +290,16 @@ URLProtocol http_protocol = {
 static char *b64_encode( unsigned char *src )
 {
     static const char b64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-                                                                                
-    char *dst = av_malloc( strlen( src ) * 4 / 3 + 12 );
-    char *ret = dst;
+    unsigned int len= strlen(src);
+    char *ret, *dst;
     unsigned i_bits = 0;
     unsigned i_shift = 0;
-                                                                                
+
+    if(len < UINT_MAX/4){
+        ret=dst= av_malloc( len * 4 / 3 + 12 );
+    }else
+        return NULL;
+
     for( ;; )
     {
         if( *src )
diff --git a/libavformat/img.c b/libavformat/img.c
index 95ab56b7e1..d52e1a0f59 100644
--- a/libavformat/img.c
+++ b/libavformat/img.c
@@ -123,7 +123,7 @@ static int img_read_header(AVFormatContext *s1, AVFormatParameters *ap)
     if (ap && ap->image_format)
         s->img_fmt = ap->image_format;
 
-    strcpy(s->path, s1->filename);
+    pstrcpy(s->path, sizeof(s->path), s1->filename);
     s->img_number = 0;
     s->img_count = 0;
     
@@ -289,7 +289,7 @@ static int img_write_header(AVFormatContext *s)
     VideoData *img = s->priv_data;
 
     img->img_number = 1;
-    strcpy(img->path, s->filename);
+    pstrcpy(img->path, sizeof(img->path), s->filename);
 
     /* find format */
     if (s->oformat->flags & AVFMT_NOFILE)
diff --git a/libavformat/img2.c b/libavformat/img2.c
index d870814b4e..fcf7310208 100644
--- a/libavformat/img2.c
+++ b/libavformat/img2.c
@@ -184,7 +184,7 @@ static int img_read_header(AVFormatContext *s1, AVFormatParameters *ap)
         return -ENOMEM;
     }
 
-    strcpy(s->path, s1->filename);
+    pstrcpy(s->path, sizeof(s->path), s1->filename);
     s->img_number = 0;
     s->img_count = 0;
     
@@ -310,7 +310,7 @@ static int img_write_header(AVFormatContext *s)
     VideoData *img = s->priv_data;
 
     img->img_number = 1;
-    strcpy(img->path, s->filename);
+    pstrcpy(img->path, sizeof(img->path), s->filename);
 
     /* find format */
     if (s->oformat->flags & AVFMT_NOFILE)
diff --git a/libavformat/matroska.c b/libavformat/matroska.c
index 9d80302f2c..1e146d464a 100644
--- a/libavformat/matroska.c
+++ b/libavformat/matroska.c
@@ -669,7 +669,7 @@ ebml_read_ascii (MatroskaDemuxContext *matroska,
 
     /* ebml strings are usually not 0-terminated, so we allocate one
      * byte more, read the string and NULL-terminate it ourselves. */
-    if (!(*str = av_malloc(size + 1))) {
+    if (size < 0 || !(*str = av_malloc(size + 1))) {
         av_log(matroska->ctx, AV_LOG_ERROR, "Memory allocation failed\n");
         return AVERROR_NOMEM;
     }
diff --git a/libavformat/mov.c b/libavformat/mov.c
index efec9d4129..6ca6a990fb 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -367,6 +367,10 @@ static int mov_read_default(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
 	    /* empty */;
 
 	a.size -= 8;
+        
+        if(a.size < 0)
+            break;
+        
 //        av_log(NULL, AV_LOG_DEBUG, " i=%ld\n", i);
 	if (c->parse_table[i].type == 0) { /* skip leaf atoms data */
 //            url_seek(pb, atom.offset+atom.size, SEEK_SET);
@@ -401,7 +405,10 @@ static int mov_read_ctab(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
 {
     unsigned int len;
     MOV_ctab_t *t;
-    //url_fskip(pb, atom.size); // for now
+#if 1
+    url_fskip(pb, atom.size); // for now
+#else
+    VERY VERY BROKEN, NEVER execute this, needs rewrite
     c->ctab = av_realloc(c->ctab, ++c->ctab_size);
     t = c->ctab[c->ctab_size];
     t->seed = get_be32(pb);
@@ -413,6 +420,7 @@ static int mov_read_ctab(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
 	if (t->clrs)
 	    get_buffer(pb, t->clrs, len);
     }
+#endif
 
     return 0;
 }
@@ -677,6 +685,9 @@ static int mov_read_smi(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
 {
     AVStream *st = c->fc->streams[c->fc->nb_streams-1];
 
+    if((uint64_t)atom.size > (1<<30))
+        return -1;
+    
     // currently SVQ3 decoder expect full STSD header - so let's fake it
     // this should be fixed and just SMI header should be passed
     av_free(st->codec.extradata);
@@ -697,6 +708,9 @@ static int mov_read_avcC(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
 {
     AVStream *st = c->fc->streams[c->fc->nb_streams-1];
 
+    if((uint64_t)atom.size > (1<<30))
+        return -1;
+
     av_free(st->codec.extradata);
 
     st->codec.extradata_size = atom.size;
@@ -714,7 +728,7 @@ static int mov_read_stco(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
 {
     AVStream *st = c->fc->streams[c->fc->nb_streams-1];
     MOVStreamContext *sc = (MOVStreamContext *)st->priv_data;
-    int entries, i;
+    unsigned int i, entries;
 
     print_atom("stco", atom);
 
@@ -722,6 +736,10 @@ static int mov_read_stco(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
     get_byte(pb); get_byte(pb); get_byte(pb); /* flags */
 
     entries = get_be32(pb);
+        
+    if(entries >= UINT_MAX/sizeof(int64_t))
+        return -1;
+ 
     sc->chunk_count = entries;
     sc->chunk_offsets = (int64_t*) av_malloc(entries * sizeof(int64_t));
     if (!sc->chunk_offsets)
@@ -1138,7 +1156,7 @@ static int mov_read_stsc(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
 {
     AVStream *st = c->fc->streams[c->fc->nb_streams-1];
     MOVStreamContext *sc = (MOVStreamContext *)st->priv_data;
-    int entries, i;
+    unsigned int i, entries;
 
     print_atom("stsc", atom);
 
@@ -1146,6 +1164,10 @@ static int mov_read_stsc(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
     get_byte(pb); get_byte(pb); get_byte(pb); /* flags */
 
     entries = get_be32(pb);
+    
+    if(entries >= UINT_MAX / sizeof(MOV_sample_to_chunk_tbl))
+        return -1;
+    
 #ifdef DEBUG
 av_log(NULL, AV_LOG_DEBUG, "track[%i].stsc.entries = %i\n", c->fc->nb_streams-1, entries);
 #endif
@@ -1168,7 +1190,7 @@ static int mov_read_stss(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
 {
     AVStream *st = c->fc->streams[c->fc->nb_streams-1];
     MOVStreamContext *sc = (MOVStreamContext *)st->priv_data;
-    int entries, i;
+    unsigned int i, entries;
 
     print_atom("stss", atom);
 
@@ -1176,6 +1198,10 @@ static int mov_read_stss(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
     get_byte(pb); get_byte(pb); get_byte(pb); /* flags */
 
     entries = get_be32(pb);
+    
+    if(entries >= UINT_MAX / sizeof(long))
+        return -1;
+    
     sc->keyframe_count = entries;
 #ifdef DEBUG
     av_log(NULL, AV_LOG_DEBUG, "keyframe_count = %ld\n", sc->keyframe_count);
@@ -1196,7 +1222,7 @@ static int mov_read_stsz(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
 {
     AVStream *st = c->fc->streams[c->fc->nb_streams-1];
     MOVStreamContext *sc = (MOVStreamContext *)st->priv_data;
-    int entries, i;
+    unsigned int i, entries;
 
     print_atom("stsz", atom);
 
@@ -1205,6 +1231,9 @@ static int mov_read_stsz(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
 
     sc->sample_size = get_be32(pb);
     entries = get_be32(pb);
+    if(entries >= UINT_MAX / sizeof(long))
+        return -1;
+
     sc->sample_count = entries;
 #ifdef DEBUG
     av_log(NULL, AV_LOG_DEBUG, "sample_size = %ld sample_count = %ld\n", sc->sample_size, sc->sample_count);
@@ -1227,7 +1256,7 @@ static int mov_read_stts(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
 {
     AVStream *st = c->fc->streams[c->fc->nb_streams-1];
     //MOVStreamContext *sc = (MOVStreamContext *)st->priv_data;
-    int entries, i;
+    unsigned int i, entries;
     int64_t duration=0;
     int64_t total_sample_count=0;
 
@@ -1236,6 +1265,8 @@ static int mov_read_stts(MOVContext *c, ByteIOContext *pb, MOV_atom_t atom)
     get_byte(pb); /* version */
     get_byte(pb); get_byte(pb); get_byte(pb); /* flags */
     entries = get_be32(pb);
+    if(entries >= UINT_MAX / sizeof(uint64_t))
+        return -1;
 
     c->streams[c->fc->nb_streams-1]->stts_count = entries;
     c->streams[c->fc->nb_streams-1]->stts_data = (uint64_t*) av_malloc(entries * sizeof(uint64_t));
diff --git a/libavformat/nsvdec.c b/libavformat/nsvdec.c
index f3831d7f39..555a71a2ed 100644
--- a/libavformat/nsvdec.c
+++ b/libavformat/nsvdec.c
@@ -365,6 +365,8 @@ static int nsv_parse_NSVf_header(AVFormatContext *s, AVFormatParameters *ap)
 
     if (table_entries_used > 0) {
         nsv->index_entries = table_entries_used;
+        if((unsigned)table_entries >= UINT_MAX / sizeof(uint32_t))
+            return -1;
         nsv->nsvf_index_data = av_malloc(table_entries * sizeof(uint32_t));
         get_buffer(pb, nsv->nsvf_index_data, table_entries * sizeof(uint32_t));
     }
diff --git a/libavformat/nut.c b/libavformat/nut.c
index 67b317ed69..010e09c3bf 100644
--- a/libavformat/nut.c
+++ b/libavformat/nut.c
@@ -82,7 +82,7 @@ typedef struct {
     int written_packet_size;
     int64_t packet_start[3]; //0-> startcode less, 1-> short startcode 2-> long startcodes
     FrameCode frame_code[256];
-    int stream_count;
+    unsigned int stream_count;
     uint64_t next_startcode;     ///< stores the next startcode if it has alraedy been parsed but the stream isnt seekable
     StreamContext *stream;
     int max_distance;
@@ -255,8 +255,8 @@ static uint64_t get_v(ByteIOContext *bc)
     return -1;
 }
 
-static int get_str(ByteIOContext *bc, char *string, int maxlen){
-    int len= get_v(bc);
+static int get_str(ByteIOContext *bc, char *string, unsigned int maxlen){
+    unsigned int len= get_v(bc);
     
     if(len && maxlen)
         get_buffer(bc, string, FFMIN(len, maxlen));
@@ -283,7 +283,7 @@ static int64_t get_s(ByteIOContext *bc){
 
 static uint64_t get_vb(ByteIOContext *bc){
     uint64_t val=0;
-    int i= get_v(bc);
+    unsigned int i= get_v(bc);
     
     if(i>8)
         return UINT64_MAX;
@@ -877,6 +877,10 @@ static int decode_main_header(NUTContext *nut){
     }
     
     nut->stream_count = get_v(bc);
+    if(nut->stream_count > MAX_STREAMS){
+        av_log(s, AV_LOG_ERROR, "too many streams\n");
+        return -1;
+    }
     nut->max_distance = get_v(bc);
     nut->max_short_distance = get_v(bc);
     nut->rate_num= get_v(bc);
@@ -982,6 +986,8 @@ static int decode_stream_header(NUTContext *nut){
     /* codec specific data headers */
     while(get_v(bc) != 0){
         st->codec.extradata_size= get_v(bc);
+        if((unsigned)st->codec.extradata_size > (1<<30))
+            return -1;
         st->codec.extradata= av_mallocz(st->codec.extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
         get_buffer(bc, st->codec.extradata, st->codec.extradata_size);            
 //	    url_fskip(bc, get_v(bc));
diff --git a/libavformat/ogg.c b/libavformat/ogg.c
index e0a72306c9..c30ccd2f24 100644
--- a/libavformat/ogg.c
+++ b/libavformat/ogg.c
@@ -195,6 +195,8 @@ static int ogg_read_header(AVFormatContext *avfcontext, AVFormatParameters *ap)
         if(next_packet(avfcontext, &op)){
             return -1;
         }
+        if(op.bytes >= (1<<16) || op.bytes < 0)
+            return -1;
         codec->extradata_size+= 2 + op.bytes;
         codec->extradata= av_realloc(codec->extradata, codec->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
         p= codec->extradata + codec->extradata_size - 2 - op.bytes;
diff --git a/libavformat/segafilm.c b/libavformat/segafilm.c
index 9e94de103c..3f752a1d9b 100644
--- a/libavformat/segafilm.c
+++ b/libavformat/segafilm.c
@@ -171,6 +171,8 @@ static int film_read_header(AVFormatContext *s,
         return AVERROR_INVALIDDATA;
     film->base_clock = BE_32(&scratch[8]);
     film->sample_count = BE_32(&scratch[12]);
+    if(film->sample_count >= UINT_MAX / sizeof(film_sample_t))
+        return -1;
     film->sample_table = av_malloc(film->sample_count * sizeof(film_sample_t));
     
     for(i=0; i<s->nb_streams; i++)
diff --git a/libavformat/sgi.c b/libavformat/sgi.c
index 13ca98e925..bbf700bc4f 100644
--- a/libavformat/sgi.c
+++ b/libavformat/sgi.c
@@ -65,6 +65,9 @@ static void read_sgi_header(ByteIOContext *f, SGIInfo *info)
     info->xsize = (unsigned short) get_be16(f);
     info->ysize = (unsigned short) get_be16(f);
     info->zsize = (unsigned short) get_be16(f);
+    
+    if(info->zsize > 4096) 
+        info->zsize= 0;
 
 #ifdef DEBUG
     printf("sgi header fields:\n");
diff --git a/libavformat/utils.c b/libavformat/utils.c
index c889b3384f..8366b35c38 100644
--- a/libavformat/utils.c
+++ b/libavformat/utils.c
@@ -180,7 +180,10 @@ static void av_destruct_packet(AVPacket *pkt)
  */
 int av_new_packet(AVPacket *pkt, int size)
 {
-    void *data = av_malloc(size + FF_INPUT_BUFFER_PADDING_SIZE);
+    void *data;
+    if((unsigned)size > (unsigned)size + FF_INPUT_BUFFER_PADDING_SIZE)
+        return AVERROR_NOMEM;        
+    data = av_malloc(size + FF_INPUT_BUFFER_PADDING_SIZE);
     if (!data)
         return AVERROR_NOMEM;
     memset(data + size, 0, FF_INPUT_BUFFER_PADDING_SIZE);
@@ -200,6 +203,8 @@ int av_dup_packet(AVPacket *pkt)
         uint8_t *data;
         /* we duplicate the packet and don't forget to put the padding
            again */
+        if((unsigned)pkt->size > (unsigned)pkt->size + FF_INPUT_BUFFER_PADDING_SIZE)
+            return AVERROR_NOMEM;        
         data = av_malloc(pkt->size + FF_INPUT_BUFFER_PADDING_SIZE);
         if (!data) {
             return AVERROR_NOMEM;
@@ -277,8 +282,8 @@ int fifo_read(FifoBuffer *f, uint8_t *buf, int buf_size, uint8_t **rptr_ptr)
     return 0;
 }
 
-void fifo_realloc(FifoBuffer *f, int new_size){
-    int old_size= f->end - f->buffer;
+void fifo_realloc(FifoBuffer *f, unsigned int new_size){
+    unsigned int old_size= f->end - f->buffer;
     
     if(old_size < new_size){
         uint8_t *old= f->buffer;
@@ -1007,10 +1012,16 @@ int av_add_index_entry(AVStream *st,
     AVIndexEntry *entries, *ie;
     int index;
     
+    if((unsigned)st->nb_index_entries + 1 >= UINT_MAX / sizeof(AVIndexEntry))
+        return -1;
+    
     entries = av_fast_realloc(st->index_entries,
                               &st->index_entries_allocated_size,
                               (st->nb_index_entries + 1) * 
                               sizeof(AVIndexEntry));
+    if(!entries)
+        return -1;
+
     st->index_entries= entries;
 
     index= av_index_search_timestamp(st, timestamp, 0);
diff --git a/libavformat/wc3movie.c b/libavformat/wc3movie.c
index 15130d97cd..b5f5c35adf 100644
--- a/libavformat/wc3movie.c
+++ b/libavformat/wc3movie.c
@@ -169,6 +169,8 @@ static int wc3_read_header(AVFormatContext *s,
             if ((ret = get_buffer(pb, preamble, 4)) != 4)
                 return AVERROR_IO;
             wc3->palette_count = LE_32(&preamble[0]);
+            if((unsigned)wc3->palette_count >= UINT_MAX / PALETTE_SIZE)
+                return -1;
             wc3->palettes = av_malloc(wc3->palette_count * PALETTE_SIZE);
             break;