fftools/ffmpeg: Take type limitations of AVFifo API into account

The types used by the AVFifo API are inconsistent:
av_fifo_(space|size)() returns an int; av_fifo_alloc() takes an
unsigned, other parts use size_t. This commit therefore ensures
that the size of the muxing_queue FIFO never exceeds INT_MAX.

While just at it, also make sure not to call av_fifo_size()
unnecessarily often.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

2 files changed, 5 insertions, 5 deletions

diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index 3761ea0c38..c838e2604c 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -753,14 +753,13 @@ static void write_packet(OutputFile *of, AVPacket *pkt, OutputStream *ost, int u
         AVPacket *tmp_pkt;
         /* the muxer is not initialized yet, buffer the packet */
         if (!av_fifo_space(ost->muxing_queue)) {
+            size_t cur_size = av_fifo_size(ost->muxing_queue);
             unsigned int are_we_over_size =
                 (ost->muxing_queue_data_size + pkt->size) > ost->muxing_queue_data_threshold;
-            int new_size = are_we_over_size ?
-                           FFMIN(2 * av_fifo_size(ost->muxing_queue),
-                                 ost->max_muxing_queue_size) :
-                           2 * av_fifo_size(ost->muxing_queue);
+            size_t limit    = are_we_over_size ? ost->max_muxing_queue_size : INT_MAX;
+            size_t new_size = FFMIN(2 * cur_size, limit);
 
-            if (new_size <= av_fifo_size(ost->muxing_queue)) {
+            if (new_size <= cur_size) {
                 av_log(NULL, AV_LOG_ERROR,
                        "Too many packets buffered for output stream %d:%d.\n",
                        ost->file_index, ost->st->index);
diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
index e55b584fd4..feed772452 100644
--- a/fftools/ffmpeg_opt.c
+++ b/fftools/ffmpeg_opt.c
@@ -1613,6 +1613,7 @@ static OutputStream *new_output_stream(OptionsContext *o, AVFormatContext *oc, e
 
     ost->max_muxing_queue_size = 128;
     MATCH_PER_STREAM_OPT(max_muxing_queue_size, i, ost->max_muxing_queue_size, oc, st);
+    ost->max_muxing_queue_size = FFMIN(ost->max_muxing_queue_size, INT_MAX / sizeof(ost->pkt));
     ost->max_muxing_queue_size *= sizeof(ost->pkt);
 
     ost->muxing_queue_data_size = 0;