lavf: improve handling of sparse streams when muxing

Currently ff_interleave_packet_per_dts() waits until it gets a frame for
each stream before outputting packets in interleaved order.

Sparse streams (i.e. streams with much fewer packets than the other
streams, like subtitles or audio with DTX) tend to add up latency and in
specific cases end up allocating a large amount of memory.

Emit the top packet from the packet_buffer if it has a time delta
larger than a specified threshold.

Original report of the issue and initial proposed solution by
mus.svz@gmail.com.

Bug-id: 31
Signed-off-by: Anton Khirnov <anton@khirnov.net>

1 files changed, 33 insertions, 2 deletions

diff --git a/libavformat/mux.c b/libavformat/mux.c
index 06dacead8f..c16d1c83a5 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -541,8 +541,39 @@ int ff_interleave_packet_per_dts(AVFormatContext *s, AVPacket *out,
         ff_interleave_add_packet(s, pkt, interleave_compare_dts);
     }
 
-    for (i = 0; i < s->nb_streams; i++)
-        stream_count += !!s->streams[i]->last_in_packet_buffer;
+    if (s->max_interleave_delta > 0 && s->packet_buffer && !flush) {
+        AVPacket *top_pkt = &s->packet_buffer->pkt;
+        int64_t delta_dts = INT64_MIN;
+        int64_t top_dts = av_rescale_q(top_pkt->dts,
+                                       s->streams[top_pkt->stream_index]->time_base,
+                                       AV_TIME_BASE_Q);
+
+        for (i = 0; i < s->nb_streams; i++) {
+            int64_t last_dts;
+            const AVPacketList *last = s->streams[i]->last_in_packet_buffer;
+
+            if (!last)
+                continue;
+
+            last_dts = av_rescale_q(last->pkt.dts,
+                                    s->streams[i]->time_base,
+                                    AV_TIME_BASE_Q);
+            delta_dts = FFMAX(delta_dts, last_dts - top_dts);
+            stream_count++;
+        }
+
+        if (delta_dts > s->max_interleave_delta) {
+            av_log(s, AV_LOG_DEBUG,
+                   "Delay between the first packet and last packet in the "
+                   "muxing queue is %"PRId64" > %"PRId64": forcing output\n",
+                   delta_dts, s->max_interleave_delta);
+            flush = 1;
+        }
+    } else {
+        for (i = 0; i < s->nb_streams; i++)
+            stream_count += !!s->streams[i]->last_in_packet_buffer;
+    }
+
 
     if (stream_count && (s->internal->nb_interleaved_streams == stream_count || flush)) {
         pktl = s->packet_buffer;