mmaldec: be more tolerant against MMAL not returning decoded output

In some situations, MMAL won't return a decoded frame for certain input
frames. This can happen if a frame fails to decode, or if a packet does
not actually contain a complete frame. In these situations, we would
deadlock (or actually timeout) waiting for an expected output frame,
which is not ideal. On the other hand, there are situations where we
definitely have to block to avoid deadlocks. (This mess is a
consequence of trying to map MMAL's asynchronous and flexible
dataflow to libavcodec, which is more static and rigid.)

Solve this by doing a blocking wait only if the amount of buffered data
is too big. The whole purpose of the blocking wait is to avoid excessive
buffering of input data, so we can skip it if it appears to be low. The
consequence is that libavcodec can gracefully return no frame to the
API user.

We want to track the number of full packets to make our heuristic work.
But MMAL buffers are fixed-size, requiring splitting large packets. This
is why the previous commit is needed. We use the ..._FRAME_END flag to
remember packet boundaries, but MMAL does not preserve these buffer
flags when returning buffers to the user.

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

1 files changed, 14 insertions, 2 deletions

diff --git a/libavcodec/mmaldec.c b/libavcodec/mmaldec.c
index da1e36cae4..15b887c71f 100644
--- a/libavcodec/mmaldec.c
+++ b/libavcodec/mmaldec.c
@@ -81,6 +81,7 @@ typedef struct MMALDecodeContext {
     FFBufferEntry *waiting_buffers, *waiting_buffers_tail;
 
     int64_t packets_sent;
+    volatile int packets_buffered;
     int64_t frames_output;
     int eos_received;
     int eos_sent;
@@ -164,6 +165,8 @@ static void ffmmal_stop_decoder(AVCodecContext *avctx)
     }
     ctx->waiting_buffers_tail = NULL;
 
+    assert(avpriv_atomic_get(&ctx->packets_buffered) == 0);
+
     ctx->frames_output = ctx->eos_received = ctx->eos_sent = ctx->packets_sent = ctx->extradata_sent = 0;
 }
 
@@ -187,9 +190,14 @@ static av_cold int ffmmal_close_decoder(AVCodecContext *avctx)
 
 static void input_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
 {
+    AVCodecContext *avctx = (AVCodecContext*)port->userdata;
+    MMALDecodeContext *ctx = avctx->priv_data;
+
     if (!buffer->cmd) {
         FFBufferEntry *entry = buffer->user_data;
         av_buffer_unref(&entry->ref);
+        if (entry->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
+            avpriv_atomic_int_add_and_fetch(&ctx->packets_buffered, -1);
         av_free(entry);
     }
     mmal_buffer_header_release(buffer);
@@ -483,8 +491,10 @@ static int ffmmal_add_packet(AVCodecContext *avctx, AVPacket *avpkt,
         buffer->pts = avpkt->pts == AV_NOPTS_VALUE ? MMAL_TIME_UNKNOWN : avpkt->pts;
         buffer->dts = avpkt->dts == AV_NOPTS_VALUE ? MMAL_TIME_UNKNOWN : avpkt->dts;
 
-        if (!size)
+        if (!size) {
             buffer->flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_END;
+            avpriv_atomic_int_add_and_fetch(&ctx->packets_buffered, 1);
+        }
 
         if (!buffer->length) {
             buffer->flags |= MMAL_BUFFER_HEADER_FLAG_EOS;
@@ -547,6 +557,8 @@ static int ffmmal_fill_input_port(AVCodecContext *avctx)
         if ((status = mmal_port_send_buffer(ctx->decoder->input[0], mbuffer))) {
             mmal_buffer_header_release(mbuffer);
             av_buffer_unref(&buffer->ref);
+            if (buffer->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
+                avpriv_atomic_int_add_and_fetch(&ctx->packets_buffered, -1);
             av_free(buffer);
         }
 
@@ -627,7 +639,7 @@ static int ffmmal_read_frame(AVCodecContext *avctx, AVFrame *frame, int *got_fra
         // excessive buffering.
         // We also wait if we sent eos, but didn't receive it yet (think of decoding
         // stream with a very low number of frames).
-        if (ctx->frames_output || ctx->packets_sent > MAX_DELAYED_FRAMES ||
+        if (avpriv_atomic_int_get(&ctx->packets_buffered) > MAX_DELAYED_FRAMES ||
             (ctx->packets_sent && ctx->eos_sent)) {
             // MMAL will ignore broken input packets, which means the frame we
             // expect here may never arrive. Dealing with this correctly is