vdpau: add helper for surface chroma type and size

Since the VDPAU pixel format does not distinguish between different
VDPAU video surface chroma types, we need another way to pass this
data to the application.

Originally VDPAU in libavcodec only supported decoding to 8-bits YUV
with 4:2:0 chroma sampling. Correspondingly, applications assumed that
libavcodec expected VDP_CHROMA_TYPE_420 video surfaces for output.
However some of the new HEVC profiles proposed for addition to VDPAU
would require different depth and/or sampling:
http://lists.freedesktop.org/archives/vdpau/2014-July/000167.html
...as would lossless AVC profiles:
http://lists.freedesktop.org/archives/vdpau/2014-November/000241.html

To preserve backward binary compatibility with existing applications,
a new av_vdpau_bind_context() flag is introduced in a further change.

Signed-off-by: Rémi Denis-Courmont <remi@remlab.net>
Signed-off-by: Anton Khirnov <anton@khirnov.net>

1 files changed, 47 insertions, 4 deletions

diff --git a/libavcodec/vdpau.c b/libavcodec/vdpau.c
index ccb3352028..ea67314f93 100644
--- a/libavcodec/vdpau.c
+++ b/libavcodec/vdpau.c
@@ -64,6 +64,46 @@ static int vdpau_error(VdpStatus status)
     }
 }
 
+int av_vdpau_get_surface_parameters(AVCodecContext *avctx,
+                                    VdpChromaType *type,
+                                    uint32_t *width, uint32_t *height)
+{
+    VdpChromaType t;
+    uint32_t w = avctx->coded_width;
+    uint32_t h = avctx->coded_height;
+
+    /* See <vdpau/vdpau.h> for per-type alignment constraints. */
+    switch (avctx->sw_pix_fmt) {
+    case AV_PIX_FMT_YUV420P:
+    case AV_PIX_FMT_YUVJ420P:
+        t = VDP_CHROMA_TYPE_420;
+        w = (w + 1) & ~1;
+        h = (h + 3) & ~3;
+        break;
+    case AV_PIX_FMT_YUV422P:
+    case AV_PIX_FMT_YUVJ422P:
+        t = VDP_CHROMA_TYPE_422;
+        w = (w + 1) & ~1;
+        h = (h + 1) & ~1;
+        break;
+    case AV_PIX_FMT_YUV444P:
+    case AV_PIX_FMT_YUVJ444P:
+        t = VDP_CHROMA_TYPE_444;
+        h = (h + 1) & ~1;
+        break;
+    default:
+        return AVERROR(ENOSYS);
+    }
+
+    if (type)
+        *type = t;
+    if (width)
+        *width = w;
+    if (height)
+        *height = h;
+    return 0;
+}
+
 int ff_vdpau_common_init(AVCodecContext *avctx, VdpDecoderProfile profile,
                          int level)
 {
@@ -76,9 +116,9 @@ int ff_vdpau_common_init(AVCodecContext *avctx, VdpDecoderProfile profile,
     VdpStatus status;
     VdpBool supported;
     uint32_t max_level, max_mb, max_width, max_height;
-    /* See vdpau/vdpau.h for alignment constraints. */
-    uint32_t width  = (avctx->coded_width + 1) & ~1;
-    uint32_t height = (avctx->coded_height + 3) & ~3;
+    VdpChromaType type;
+    uint32_t width;
+    uint32_t height;
 
     vdctx->width            = UINT32_MAX;
     vdctx->height           = UINT32_MAX;
@@ -99,6 +139,9 @@ int ff_vdpau_common_init(AVCodecContext *avctx, VdpDecoderProfile profile,
     else if (level < 0)
         return AVERROR(ENOTSUP);
 
+    if (av_vdpau_get_surface_parameters(avctx, &type, &width, &height))
+        return AVERROR(ENOSYS);
+
     status = vdctx->get_proc_address(vdctx->device,
                                      VDP_FUNC_ID_VIDEO_SURFACE_QUERY_CAPABILITIES,
                                      &func);
@@ -107,7 +150,7 @@ int ff_vdpau_common_init(AVCodecContext *avctx, VdpDecoderProfile profile,
     else
         surface_query_caps = func;
 
-    status = surface_query_caps(vdctx->device, VDP_CHROMA_TYPE_420, &supported,
+    status = surface_query_caps(vdctx->device, type, &supported,
                                 &max_width, &max_height);
     if (status != VDP_STATUS_OK)
         return vdpau_error(status);