Merge remote-tracking branch 'qatar/master'

* qatar/master:
  movenc: fix NULL reference in mov_write_tkhd_tag
  rmdec: Reject invalid deinterleaving parameters
  rv34: Fix potential overreads
  rv34: Fix buffer size used for MC of B frames after a resolution change
  rv34: Avoid NULL dereference on corrupted bitstream
  rv10: Reject slices that does not have the same type as the first one
  vf_yadif: add an option to enable/disable deinterlacing based on src frame "interlaced" flag
  vsrc_color: set output pos values to -1
  vsrc_color: add @file doxy
  vsrc_buffer: remove duplicated file description
  eval: implement not() expression
  eval: add sqrt function for computing the square root
  rmdec: use the deinterleaving mode and not the codec when creating audio packets.
  lavf: Fix context pointer in av_open_input_stream when avformat_open_input fails

Conflicts:
	doc/eval.texi
	doc/filters.texi
	libavcodec/rv10.c
	libavfilter/vsrc_color.c
	libavformat/rmdec.c
	libavutil/avutil.h
	libavutil/eval.c
	tests/ref/fate/eval

Merged-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 33 insertions, 28 deletions

diff --git a/libavformat/rmdec.c b/libavformat/rmdec.c
index 4d1e75aa40..131345cdc4 100644
--- a/libavformat/rmdec.c
+++ b/libavformat/rmdec.c
@@ -194,18 +194,6 @@ static int rm_read_audio_stream_info(AVFormatContext *s, AVIOContext *pb,
         st->codec->codec_id   = ff_codec_get_id(ff_rm_codec_tags,
                                                 st->codec->codec_tag);
 
-        switch (ast->deint_id) {
-        case DEINT_ID_GENR:
-        case DEINT_ID_INT0:
-        case DEINT_ID_INT4:
-        case DEINT_ID_SIPR:
-        case DEINT_ID_VBRS:
-        case DEINT_ID_VBRF:
-            break;
-        default:
-            av_log(NULL,0,"Unknown interleaver %X\n", ast->deint_id);
-            return AVERROR_INVALIDDATA;
-        }
         switch (st->codec->codec_id) {
         case CODEC_ID_AC3:
             st->need_parsing = AVSTREAM_PARSE_FULL;
@@ -214,14 +202,6 @@ static int rm_read_audio_stream_info(AVFormatContext *s, AVIOContext *pb,
             st->codec->extradata_size= 0;
             ast->audio_framesize = st->codec->block_align;
             st->codec->block_align = coded_framesize;
-
-            if (ast->audio_framesize <= 0 || sub_packet_h <= 0 ||
-                ast->audio_framesize >= UINT_MAX / sub_packet_h){
-                av_log(s, AV_LOG_ERROR, "ast->audio_framesize * sub_packet_h is invalid\n");
-                return -1;
-            }
-
-            av_new_packet(&ast->pkt, ast->audio_framesize * sub_packet_h);
             break;
         case CODEC_ID_COOK:
         case CODEC_ID_ATRAC3:
@@ -253,13 +233,6 @@ static int rm_read_audio_stream_info(AVFormatContext *s, AVIOContext *pb,
             if ((ret = rm_read_extradata(pb, st->codec, codecdata_length)) < 0)
                 return ret;
 
-            if (ast->audio_framesize <= 0 || sub_packet_h <= 0 ||
-                ast->audio_framesize >= UINT_MAX / sub_packet_h){
-                av_log(s, AV_LOG_ERROR, "rm->audio_framesize * sub_packet_h is invalid\n");
-                return -1;
-            }
-
-            av_new_packet(&ast->pkt, ast->audio_framesize * sub_packet_h);
             break;
         case CODEC_ID_AAC:
             avio_rb16(pb); avio_r8(pb);
@@ -279,6 +252,37 @@ static int rm_read_audio_stream_info(AVFormatContext *s, AVIOContext *pb,
         default:
             av_strlcpy(st->codec->codec_name, buf, sizeof(st->codec->codec_name));
         }
+        if (ast->deint_id == DEINT_ID_INT4 ||
+            ast->deint_id == DEINT_ID_GENR ||
+            ast->deint_id == DEINT_ID_SIPR) {
+            if (st->codec->block_align <= 0 ||
+                ast->audio_framesize * sub_packet_h > (unsigned)INT_MAX ||
+                ast->audio_framesize * sub_packet_h < st->codec->block_align)
+                return AVERROR_INVALIDDATA;
+            if (av_new_packet(&ast->pkt, ast->audio_framesize * sub_packet_h) < 0)
+                return AVERROR(ENOMEM);
+        }
+        switch (ast->deint_id) {
+        case DEINT_ID_INT4:
+            if (ast->coded_framesize > ast->audio_framesize ||
+                ast->coded_framesize * sub_packet_h > (2 + (sub_packet_h & 1)) * ast->audio_framesize)
+                return AVERROR_INVALIDDATA;
+            break;
+        case DEINT_ID_GENR:
+            if (ast->sub_packet_size <= 0 ||
+                ast->sub_packet_size > ast->audio_framesize)
+                return AVERROR_INVALIDDATA;
+            break;
+        case DEINT_ID_SIPR:
+        case DEINT_ID_INT0:
+        case DEINT_ID_VBRS:
+        case DEINT_ID_VBRF:
+            break;
+        default:
+            av_log(NULL,0,"Unknown interleaver %X\n", ast->deint_id);
+            return AVERROR_INVALIDDATA;
+        }
+
         if (read_all) {
             avio_r8(pb);
             avio_r8(pb);
@@ -815,7 +819,8 @@ ff_rm_retrieve_cache (AVFormatContext *s, AVIOContext *pb,
 
     assert (rm->audio_pkt_cnt > 0);
 
-    if (st->codec->codec_id == CODEC_ID_AAC)
+    if (ast->deint_id == DEINT_ID_VBRF ||
+        ast->deint_id == DEINT_ID_VBRS)
         av_get_packet(pb, pkt, ast->sub_packet_lengths[ast->sub_packet_cnt - rm->audio_pkt_cnt]);
     else {
         av_new_packet(pkt, st->codec->block_align);