h264_mp4toannexb: Remove unnecessary check

There can be at most 31 SPS and 255 PPS in the mp4/Matroska extradata.
Given that each has a size of at most 2^16-1, the length of the output
derived from these parameter sets can never overflow an ordinary 32 bit
integer. So use a simple uint32_t instead of uint64_t and replace the
unnecessary check with an av_assert1.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

1 files changed, 3 insertions, 7 deletions

diff --git a/libavcodec/h264_mp4toannexb_bsf.c b/libavcodec/h264_mp4toannexb_bsf.c
index fb3f24ea40..bbf124ad04 100644
--- a/libavcodec/h264_mp4toannexb_bsf.c
+++ b/libavcodec/h264_mp4toannexb_bsf.c
@@ -21,6 +21,7 @@
 
 #include <string.h>
 
+#include "libavutil/avassert.h"
 #include "libavutil/intreadwrite.h"
 #include "libavutil/mem.h"
 
@@ -68,7 +69,7 @@ static int h264_extradata_to_annexb(AVBSFContext *ctx, const int padding)
 {
     H264BSFContext *s = ctx->priv_data;
     uint16_t unit_size;
-    uint64_t total_size                 = 0;
+    uint32_t total_size                 = 0;
     uint8_t *out                        = NULL, unit_nb, sps_done = 0,
              sps_seen                   = 0, pps_seen = 0;
     const uint8_t *extradata            = ctx->par_in->extradata + 4;
@@ -91,12 +92,7 @@ static int h264_extradata_to_annexb(AVBSFContext *ctx, const int padding)
 
         unit_size   = AV_RB16(extradata);
         total_size += unit_size + 4;
-        if (total_size > INT_MAX - padding) {
-            av_log(ctx, AV_LOG_ERROR,
-                   "Too big extradata size, corrupted stream or invalid MP4/AVCC bitstream\n");
-            av_free(out);
-            return AVERROR(EINVAL);
-        }
+        av_assert1(total_size <= INT_MAX - padding);
         if (extradata + 2 + unit_size > ctx->par_in->extradata + ctx->par_in->extradata_size) {
             av_log(ctx, AV_LOG_ERROR, "Packet header is not contained in global extradata, "
                    "corrupted stream or invalid MP4/AVCC bitstream\n");