avcodec/v4l2_m2m_enc: Check encoder pix_fmt matches pix_fmt on device

Fixes #8079

During initialization of a v4l2m2m device, the configured pix_fmt can be
different to the pix_fmt of the encoder (i.e. avctx->pix_fmt).

For example on the Odroid XU4:
./ffmpeg -f lavfi -i yuvtestsrc -codec:v h264_v4l2m2m out.h264

will configure the v4l2 encoder to pix_fmt nv21, whereas the input
frames will be yuv444p.

This commit checks that the configured v4l2 pix_fmt on device is the
same as avctx->pix_fmt. If they are different the initialization fails
and an error is returned. Tested on RPI4 and Odroid XU4.

Signed-off-by: Andriy Gelman <andriy.gelman@gmail.com>

1 files changed, 15 insertions, 0 deletions

diff --git a/libavcodec/v4l2_m2m_enc.c b/libavcodec/v4l2_m2m_enc.c
index 474e6bef89..8059e3bb48 100644
--- a/libavcodec/v4l2_m2m_enc.c
+++ b/libavcodec/v4l2_m2m_enc.c
@@ -30,6 +30,7 @@
 #include "libavutil/opt.h"
 #include "v4l2_context.h"
 #include "v4l2_m2m.h"
+#include "v4l2_fmt.h"
 
 #define MPEG_CID(x) V4L2_CID_MPEG_VIDEO_##x
 #define MPEG_VIDEO(x) V4L2_MPEG_VIDEO_##x
@@ -288,6 +289,8 @@ static av_cold int v4l2_encode_init(AVCodecContext *avctx)
     V4L2Context *capture, *output;
     V4L2m2mContext *s;
     V4L2m2mPriv *priv = avctx->priv_data;
+    enum AVPixelFormat pix_fmt_output;
+    uint32_t v4l2_fmt_output;
     int ret;
 
     ret = ff_v4l2_m2m_create_context(priv, &s);
@@ -316,6 +319,18 @@ static av_cold int v4l2_encode_init(AVCodecContext *avctx)
     }
     s->avctx = avctx;
 
+    if (V4L2_TYPE_IS_MULTIPLANAR(output->type))
+        v4l2_fmt_output = output->format.fmt.pix_mp.pixelformat;
+    else
+        v4l2_fmt_output = output->format.fmt.pix.pixelformat;
+
+    pix_fmt_output = ff_v4l2_format_v4l2_to_avfmt(v4l2_fmt_output, AV_CODEC_ID_RAWVIDEO);
+    if (pix_fmt_output != avctx->pix_fmt) {
+        const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt_output);
+        av_log(avctx, AV_LOG_ERROR, "Encoder requires %s pixel format.\n", desc->name);
+        return AVERROR(EINVAL);
+    }
+
     return v4l2_prepare_encoder(s);
 }