avisynth: support pix_fmts added to AviSynth+

A number of new pix_fmts have been added to AviSynth+:
16-bit packed RGB and RGBA
10-, 12-, 14, and 16-bit YUV 4:2:0, 4:2:2, and 4:4:4
8-, 10-, 12-, 14-, and 16-bit Planar RGB
8-, 10-, 12-, 14-, and 16-bit Planar YUVA and Planar RGBA*
10-, 12-, 14-, and 16-bit GRAY variants*
32-bit floating point Planar YUV(A), Planar RGB(A), and GRAY*

*some of which are not currently available pix_fmts here and were
 not added to the demuxer due to this

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

1 files changed, 182 insertions, 1 deletions

diff --git a/libavformat/avisynth.c b/libavformat/avisynth.c
index 04ac2576cc..1fe8e083be 100644
--- a/libavformat/avisynth.c
+++ b/libavformat/avisynth.c
@@ -68,6 +68,8 @@ typedef struct AviSynthLibrary {
     AVSC_DECLARE_FUNC(avs_get_pitch_p);
     AVSC_DECLARE_FUNC(avs_get_read_ptr_p);
     AVSC_DECLARE_FUNC(avs_get_row_size_p);
+    AVSC_DECLARE_FUNC(avs_is_planar_rgb);
+    AVSC_DECLARE_FUNC(avs_is_planar_rgba);
 #endif
 #undef AVSC_DECLARE_FUNC
 } AviSynthLibrary;
@@ -95,6 +97,14 @@ static const int avs_planes_packed[1] = { 0 };
 static const int avs_planes_grey[1]   = { AVS_PLANAR_Y };
 static const int avs_planes_yuv[3]    = { AVS_PLANAR_Y, AVS_PLANAR_U,
                                           AVS_PLANAR_V };
+#ifdef USING_AVISYNTH
+static const int avs_planes_rgb[3]    = { AVS_PLANAR_G, AVS_PLANAR_B,
+                                          AVS_PLANAR_R };
+static const int avs_planes_yuva[4]   = { AVS_PLANAR_Y, AVS_PLANAR_U,
+                                          AVS_PLANAR_V, AVS_PLANAR_A };
+static const int avs_planes_rgba[4]   = { AVS_PLANAR_G, AVS_PLANAR_B,
+                                          AVS_PLANAR_R, AVS_PLANAR_A };
+#endif
 
 /* A conflict between C++ global objects, atexit, and dynamic loading requires
  * us to register our own atexit handler to prevent double freeing. */
@@ -138,6 +148,8 @@ static av_cold int avisynth_load_library(void)
     LOAD_AVS_FUNC(avs_get_pitch_p, 1);
     LOAD_AVS_FUNC(avs_get_read_ptr_p, 1);
     LOAD_AVS_FUNC(avs_get_row_size_p, 1);
+    LOAD_AVS_FUNC(avs_is_planar_rgb, 1);
+    LOAD_AVS_FUNC(avs_is_planar_rgba, 1);
 #endif
 #undef LOAD_AVS_FUNC
 
@@ -222,7 +234,7 @@ static av_cold void avisynth_atexit_handler(void)
 static int avisynth_create_stream_video(AVFormatContext *s, AVStream *st)
 {
     AviSynthContext *avs = s->priv_data;
-    int planar = 0; // 0: packed, 1: YUV, 2: Y8
+    int planar = 0; // 0: packed, 1: YUV, 2: Y8, 3: Planar RGB, 4: YUVA, 5: Planar RGBA
 
     st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
     st->codecpar->codec_id   = AV_CODEC_ID_RAWVIDEO;
@@ -238,6 +250,136 @@ static int avisynth_create_stream_video(AVFormatContext *s, AVStream *st)
 
     switch (avs->vi->pixel_type) {
 #ifdef USING_AVISYNTH
+/* 10~16-bit YUV pix_fmts (AviSynth+) */
+    case AVS_CS_YUV444P10:
+        st->codecpar->format = AV_PIX_FMT_YUV444P10;
+        planar               = 1;
+        break;
+    case AVS_CS_YUV422P10:
+        st->codecpar->format = AV_PIX_FMT_YUV422P10;
+        planar               = 1;
+        break;
+    case AVS_CS_YUV420P10:
+        st->codecpar->format = AV_PIX_FMT_YUV420P10;
+        planar               = 1;
+        break;
+    case AVS_CS_YUV444P12:
+        st->codecpar->format = AV_PIX_FMT_YUV444P12;
+        planar               = 1;
+        break;
+    case AVS_CS_YUV422P12:
+        st->codecpar->format = AV_PIX_FMT_YUV422P12;
+        planar               = 1;
+        break;
+    case AVS_CS_YUV420P12:
+        st->codecpar->format = AV_PIX_FMT_YUV420P12;
+        planar               = 1;
+        break;
+    case AVS_CS_YUV444P14:
+        st->codecpar->format = AV_PIX_FMT_YUV444P14;
+        planar               = 1;
+        break;
+    case AVS_CS_YUV422P14:
+        st->codecpar->format = AV_PIX_FMT_YUV422P14;
+        planar               = 1;
+        break;
+    case AVS_CS_YUV420P14:
+        st->codecpar->format = AV_PIX_FMT_YUV420P14;
+        planar               = 1;
+        break;
+    case AVS_CS_YUV444P16:
+        st->codecpar->format = AV_PIX_FMT_YUV444P16;
+        planar               = 1;
+        break;
+    case AVS_CS_YUV422P16:
+        st->codecpar->format = AV_PIX_FMT_YUV422P16;
+        planar               = 1;
+        break;
+    case AVS_CS_YUV420P16:
+        st->codecpar->format = AV_PIX_FMT_YUV420P16;
+        planar               = 1;
+        break;
+/* 8~16-bit YUV pix_fmts with Alpha (AviSynth+) */
+    case AVS_CS_YUVA444:
+        st->codecpar->format = AV_PIX_FMT_YUVA444P;
+        planar               = 4;
+        break;
+    case AVS_CS_YUVA422:
+        st->codecpar->format = AV_PIX_FMT_YUVA422P;
+        planar               = 4;
+        break;
+    case AVS_CS_YUVA420:
+        st->codecpar->format = AV_PIX_FMT_YUVA420P;
+        planar               = 4;
+        break;
+    case AVS_CS_YUVA444P10:
+        st->codecpar->format = AV_PIX_FMT_YUVA444P10;
+        planar               = 4;
+        break;
+    case AVS_CS_YUVA422P10:
+        st->codecpar->format = AV_PIX_FMT_YUVA422P10;
+        planar               = 4;
+        break;
+    case AVS_CS_YUVA420P10:
+        st->codecpar->format = AV_PIX_FMT_YUVA420P10;
+        planar               = 4;
+        break;
+    case AVS_CS_YUVA444P16:
+        st->codecpar->format = AV_PIX_FMT_YUVA444P16;
+        planar               = 4;
+        break;
+    case AVS_CS_YUVA422P16:
+        st->codecpar->format = AV_PIX_FMT_YUVA422P16;
+        planar               = 4;
+        break;
+    case AVS_CS_YUVA420P16:
+        st->codecpar->format = AV_PIX_FMT_YUVA420P16;
+        planar               = 4;
+        break;
+/* Planar RGB pix_fmts (AviSynth+)  */
+    case AVS_CS_RGBP:
+        st->codecpar->format = AV_PIX_FMT_GBRP;
+        planar               = 3;
+        break;
+    case AVS_CS_RGBP10:
+        st->codecpar->format = AV_PIX_FMT_GBRP10;
+        planar               = 3;
+        break;
+    case AVS_CS_RGBP12:
+        st->codecpar->format = AV_PIX_FMT_GBRP12;
+        planar               = 3;
+        break;
+    case AVS_CS_RGBP14:
+        st->codecpar->format = AV_PIX_FMT_GBRP14;
+        planar               = 3;
+        break;
+    case AVS_CS_RGBP16:
+        st->codecpar->format = AV_PIX_FMT_GBRP16;
+        planar               = 3;
+        break;
+/* Planar RGB pix_fmts with Alpha (AviSynth+) */
+    case AVS_CS_RGBAP:
+        st->codecpar->format = AV_PIX_FMT_GBRAP;
+        planar               = 5;
+        break;
+    case AVS_CS_RGBAP10:
+        st->codecpar->format = AV_PIX_FMT_GBRAP10;
+        planar               = 5;
+        break;
+    case AVS_CS_RGBAP12:
+        st->codecpar->format = AV_PIX_FMT_GBRAP12;
+        planar               = 5;
+        break;
+    case AVS_CS_RGBAP16:
+        st->codecpar->format = AV_PIX_FMT_GBRAP16;
+        planar               = 5;
+        break;
+/* GRAY16 (AviSynth+) */
+    case AVS_CS_Y16:
+        st->codecpar->format = AV_PIX_FMT_GRAY16;
+        planar               = 2;
+        break;
+/* pix_fmts added in AviSynth 2.6 */
     case AVS_CS_YV24:
         st->codecpar->format = AV_PIX_FMT_YUV444P;
         planar               = 1;
@@ -254,7 +396,15 @@ static int avisynth_create_stream_video(AVFormatContext *s, AVStream *st)
         st->codecpar->format = AV_PIX_FMT_GRAY8;
         planar               = 2;
         break;
+/* 16-bit packed RGB pix_fmts (AviSynth+) */
+    case AVS_CS_BGR48:
+        st->codecpar->format = AV_PIX_FMT_BGR48;
+        break;
+    case AVS_CS_BGR64:
+        st->codecpar->format = AV_PIX_FMT_BGRA64;
+        break;
 #endif
+/* AviSynth 2.5 and AvxSynth pix_fmts */
     case AVS_CS_BGR24:
         st->codecpar->format = AV_PIX_FMT_BGR24;
         break;
@@ -280,6 +430,20 @@ static int avisynth_create_stream_video(AVFormatContext *s, AVStream *st)
     }
 
     switch (planar) {
+#ifdef USING_AVISYNTH
+    case 5: // Planar RGB + Alpha
+        avs->n_planes = 4;
+        avs->planes   = avs_planes_rgba;
+        break;
+    case 4: // YUV + Alpha
+        avs->n_planes = 4;
+        avs->planes   = avs_planes_yuva;
+        break;
+    case 3: // Planar RGB
+        avs->n_planes = 3;
+        avs->planes   = avs_planes_rgb;
+        break;
+#endif
     case 2: // Y8
         avs->n_planes = 1;
         avs->planes   = avs_planes_grey;
@@ -457,6 +621,16 @@ static int avisynth_read_packet_video(AVFormatContext *s, AVPacket *pkt,
         return 0;
 
 #ifdef USING_AVISYNTH
+    /* Detect whether we're using AviSynth 2.6 or AviSynth+ by
+     * looking for whether avs_is_planar_rgb exists. */
+
+    int avsplus;
+
+    if (GetProcAddress(avs_library.library, "avs_is_planar_rgb") == NULL)
+        avsplus = 0;
+    else
+        avsplus = 1;
+
     /* avs_bits_per_pixel changed to AVSC_API with AviSynth 2.6, which
      * requires going through avs_library, while AvxSynth has it under
      * the older AVSC_INLINE type, so special-case this. */
@@ -513,6 +687,13 @@ static int avisynth_read_packet_video(AVFormatContext *s, AVPacket *pkt,
             pitch = -pitch;
         }
 
+#ifdef USING_AVISYNTH
+        /* Flip Planar RGB video. */
+        if (avsplus && (avs_library.avs_is_planar_rgb(avs->vi) ||
+                        avs_library.avs_is_planar_rgba(avs->vi)))
+            pitch = -pitch;
+#endif
+
         avs_library.avs_bit_blt(avs->env, dst_p, rowsize, src_p, pitch,
                                  rowsize, planeheight);
         dst_p += rowsize * planeheight;