From 532b23f079b52f4789be1f20ce232286ce4ffa13 Mon Sep 17 00:00:00 2001 From: wm4 Date: Mon, 15 May 2017 11:27:24 +0200 Subject: videotoolbox: add hwcontext support This adds tons of code for no other benefit than making VideoToolbox support conform with the new hwaccel API (using hw_device_ctx and hw_frames_ctx). Since VideoToolbox decoding does not actually require the user to allocate frames, the new code does mostly nothing. One benefit is that ffmpeg_videotoolbox.c can be dropped once generic hwaccel support for ffmpeg.c is merged from Libav. Does not consider VDA or VideoToolbox encoding. Fun fact: the frame transfer functions are copied from vaapi, as the mapping makes copying generic boilerplate. Mapping itself is not exported by the VT code, because I don't know how to test. --- libavutil/Makefile | 3 + libavutil/hwcontext.c | 3 + libavutil/hwcontext.h | 1 + libavutil/hwcontext_internal.h | 1 + libavutil/hwcontext_videotoolbox.c | 243 +++++++++++++++++++++++++++++++++++++ libavutil/hwcontext_videotoolbox.h | 54 +++++++++ libavutil/version.h | 2 +- 7 files changed, 306 insertions(+), 1 deletion(-) create mode 100644 libavutil/hwcontext_videotoolbox.c create mode 100644 libavutil/hwcontext_videotoolbox.h (limited to 'libavutil') diff --git a/libavutil/Makefile b/libavutil/Makefile index 1fd8dca54b..92ee5bd0b3 100644 --- a/libavutil/Makefile +++ b/libavutil/Makefile @@ -36,6 +36,7 @@ HEADERS = adler32.h \ hwcontext_dxva2.h \ hwcontext_qsv.h \ hwcontext_vaapi.h \ + hwcontext_videotoolbox.h \ hwcontext_vdpau.h \ imgutils.h \ intfloat.h \ @@ -160,6 +161,7 @@ OBJS-$(CONFIG_QSV) += hwcontext_qsv.o OBJS-$(CONFIG_LZO) += lzo.o OBJS-$(CONFIG_OPENCL) += opencl.o opencl_internal.o OBJS-$(CONFIG_VAAPI) += hwcontext_vaapi.o +OBJS-$(CONFIG_VIDEOTOOLBOX) += hwcontext_videotoolbox.o OBJS-$(CONFIG_VDPAU) += hwcontext_vdpau.o OBJS += $(COMPAT_OBJS:%=../compat/%) @@ -172,6 +174,7 @@ SKIPHEADERS-$(CONFIG_CUDA) += hwcontext_cuda_internal.h SKIPHEADERS-$(CONFIG_DXVA2) += hwcontext_dxva2.h SKIPHEADERS-$(CONFIG_QSV) += hwcontext_qsv.h SKIPHEADERS-$(CONFIG_VAAPI) += hwcontext_vaapi.h +SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX) += hwcontext_videotoolbox.h SKIPHEADERS-$(CONFIG_VDPAU) += hwcontext_vdpau.h SKIPHEADERS-$(HAVE_ATOMICS_GCC) += atomic_gcc.h SKIPHEADERS-$(HAVE_ATOMICS_SUNCC) += atomic_suncc.h diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c index 4cfe377982..8d50a32b84 100644 --- a/libavutil/hwcontext.c +++ b/libavutil/hwcontext.c @@ -43,6 +43,9 @@ static const HWContextType *hw_table[] = { #endif #if CONFIG_VDPAU &ff_hwcontext_type_vdpau, +#endif +#if CONFIG_VIDEOTOOLBOX + &ff_hwcontext_type_videotoolbox, #endif NULL, }; diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h index 284b091209..cfc6ad0e28 100644 --- a/libavutil/hwcontext.h +++ b/libavutil/hwcontext.h @@ -30,6 +30,7 @@ enum AVHWDeviceType { AV_HWDEVICE_TYPE_VAAPI, AV_HWDEVICE_TYPE_DXVA2, AV_HWDEVICE_TYPE_QSV, + AV_HWDEVICE_TYPE_VIDEOTOOLBOX, }; typedef struct AVHWDeviceInternal AVHWDeviceInternal; diff --git a/libavutil/hwcontext_internal.h b/libavutil/hwcontext_internal.h index 30fce2afd9..cf05323e15 100644 --- a/libavutil/hwcontext_internal.h +++ b/libavutil/hwcontext_internal.h @@ -144,5 +144,6 @@ extern const HWContextType ff_hwcontext_type_dxva2; extern const HWContextType ff_hwcontext_type_qsv; extern const HWContextType ff_hwcontext_type_vaapi; extern const HWContextType ff_hwcontext_type_vdpau; +extern const HWContextType ff_hwcontext_type_videotoolbox; #endif /* AVUTIL_HWCONTEXT_INTERNAL_H */ diff --git a/libavutil/hwcontext_videotoolbox.c b/libavutil/hwcontext_videotoolbox.c new file mode 100644 index 0000000000..cc00f1f2f2 --- /dev/null +++ b/libavutil/hwcontext_videotoolbox.c @@ -0,0 +1,243 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" + +#include +#include + +#include + +#include "buffer.h" +#include "common.h" +#include "hwcontext.h" +#include "hwcontext_internal.h" +#include "hwcontext_videotoolbox.h" +#include "mem.h" +#include "pixfmt.h" +#include "pixdesc.h" + +static const struct { + uint32_t cv_fmt; + enum AVPixelFormat pix_fmt; +} cv_pix_fmts[] = { + { kCVPixelFormatType_420YpCbCr8Planar, AV_PIX_FMT_YUV420P }, + { kCVPixelFormatType_422YpCbCr8, AV_PIX_FMT_UYVY422 }, + { kCVPixelFormatType_32BGRA, AV_PIX_FMT_BGRA }, +#ifdef kCFCoreFoundationVersionNumber10_7 + { kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, AV_PIX_FMT_NV12 }, +#endif +}; + +enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt) +{ + int i; + for (i = 0; i < FF_ARRAY_ELEMS(cv_pix_fmts); i++) { + if (cv_pix_fmts[i].cv_fmt == cv_fmt) + return cv_pix_fmts[i].pix_fmt; + } + return AV_PIX_FMT_NONE; +} + +uint32_t av_map_videotoolbox_format_from_pixfmt(enum AVPixelFormat pix_fmt) +{ + int i; + for (i = 0; i < FF_ARRAY_ELEMS(cv_pix_fmts); i++) { + if (cv_pix_fmts[i].pix_fmt == pix_fmt) + return cv_pix_fmts[i].cv_fmt; + } + return 0; +} + +static int vt_get_buffer(AVHWFramesContext *ctx, AVFrame *frame) +{ + frame->buf[0] = av_buffer_pool_get(ctx->pool); + if (!frame->buf[0]) + return AVERROR(ENOMEM); + + frame->data[3] = frame->buf[0]->data; + frame->format = AV_PIX_FMT_VIDEOTOOLBOX; + frame->width = ctx->width; + frame->height = ctx->height; + + return 0; +} + +static int vt_transfer_get_formats(AVHWFramesContext *ctx, + enum AVHWFrameTransferDirection dir, + enum AVPixelFormat **formats) +{ + enum AVPixelFormat *fmts = av_malloc_array(2, sizeof(*fmts)); + if (!fmts) + return AVERROR(ENOMEM); + + fmts[0] = ctx->sw_format; + fmts[1] = AV_PIX_FMT_NONE; + + *formats = fmts; + return 0; +} + +static void vt_unmap(AVHWFramesContext *ctx, HWMapDescriptor *hwmap) +{ + CVPixelBufferRef pixbuf = (CVPixelBufferRef)hwmap->source->data[3]; + + CVPixelBufferUnlockBaseAddress(pixbuf, (uintptr_t)hwmap->priv); +} + +static int vt_map_frame(AVHWFramesContext *ctx, AVFrame *dst, const AVFrame *src, + int flags) +{ + CVPixelBufferRef pixbuf = (CVPixelBufferRef)src->data[3]; + OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf); + CVReturn err; + uint32_t map_flags = 0; + int ret; + int i; + enum AVPixelFormat format; + + format = av_map_videotoolbox_format_to_pixfmt(pixel_format); + if (dst->format != format) { + av_log(ctx, AV_LOG_ERROR, "Unsupported or mismatching pixel format: %s\n", + av_fourcc2str(pixel_format)); + return AVERROR_UNKNOWN; + } + + if (CVPixelBufferGetWidth(pixbuf) != ctx->width || + CVPixelBufferGetHeight(pixbuf) != ctx->height) { + av_log(ctx, AV_LOG_ERROR, "Inconsistent frame dimensions.\n"); + return AVERROR_UNKNOWN; + } + + if (flags == AV_HWFRAME_MAP_READ) + map_flags = kCVPixelBufferLock_ReadOnly; + + err = CVPixelBufferLockBaseAddress(pixbuf, map_flags); + if (err != kCVReturnSuccess) { + av_log(ctx, AV_LOG_ERROR, "Error locking the pixel buffer.\n"); + return AVERROR_UNKNOWN; + } + + if (CVPixelBufferIsPlanar(pixbuf)) { + int planes = CVPixelBufferGetPlaneCount(pixbuf); + for (i = 0; i < planes; i++) { + dst->data[i] = CVPixelBufferGetBaseAddressOfPlane(pixbuf, i); + dst->linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(pixbuf, i); + } + } else { + dst->data[0] = CVPixelBufferGetBaseAddress(pixbuf); + dst->linesize[0] = CVPixelBufferGetBytesPerRow(pixbuf); + } + + ret = ff_hwframe_map_create(src->hw_frames_ctx, dst, src, vt_unmap, + (void *)(uintptr_t)map_flags); + if (ret < 0) + goto unlock; + + return 0; + +unlock: + CVPixelBufferUnlockBaseAddress(pixbuf, map_flags); + return ret; +} + +static int vt_transfer_data_from(AVHWFramesContext *hwfc, + AVFrame *dst, const AVFrame *src) +{ + AVFrame *map; + int err; + + if (dst->width > hwfc->width || dst->height > hwfc->height) + return AVERROR(EINVAL); + + map = av_frame_alloc(); + if (!map) + return AVERROR(ENOMEM); + map->format = dst->format; + + err = vt_map_frame(hwfc, map, src, AV_HWFRAME_MAP_READ); + if (err) + goto fail; + + map->width = dst->width; + map->height = dst->height; + + err = av_frame_copy(dst, map); + if (err) + goto fail; + + err = 0; +fail: + av_frame_free(&map); + return err; +} + +static int vt_transfer_data_to(AVHWFramesContext *hwfc, + AVFrame *dst, const AVFrame *src) +{ + AVFrame *map; + int err; + + if (src->width > hwfc->width || src->height > hwfc->height) + return AVERROR(EINVAL); + + map = av_frame_alloc(); + if (!map) + return AVERROR(ENOMEM); + map->format = src->format; + + err = vt_map_frame(hwfc, map, dst, AV_HWFRAME_MAP_WRITE | AV_HWFRAME_MAP_OVERWRITE); + if (err) + goto fail; + + map->width = src->width; + map->height = src->height; + + err = av_frame_copy(map, src); + if (err) + goto fail; + + err = 0; +fail: + av_frame_free(&map); + return err; +} + +static int vt_device_create(AVHWDeviceContext *ctx, const char *device, + AVDictionary *opts, int flags) +{ + if (device && device[0]) { + av_log(ctx, AV_LOG_ERROR, "Device selection unsupported.\n"); + return AVERROR_UNKNOWN; + } + + return 0; +} + +const HWContextType ff_hwcontext_type_videotoolbox = { + .type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX, + .name = "videotoolbox", + + .device_create = vt_device_create, + .frames_get_buffer = vt_get_buffer, + .transfer_get_formats = vt_transfer_get_formats, + .transfer_data_to = vt_transfer_data_to, + .transfer_data_from = vt_transfer_data_from, + + .pix_fmts = (const enum AVPixelFormat[]){ AV_PIX_FMT_VIDEOTOOLBOX, AV_PIX_FMT_NONE }, +}; diff --git a/libavutil/hwcontext_videotoolbox.h b/libavutil/hwcontext_videotoolbox.h new file mode 100644 index 0000000000..7b4de4b7e1 --- /dev/null +++ b/libavutil/hwcontext_videotoolbox.h @@ -0,0 +1,54 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVUTIL_HWCONTEXT_VT_H +#define AVUTIL_HWCONTEXT_VT_H + +#include + +#include + +#include "pixfmt.h" + +/** + * @file + * An API-specific header for AV_HWDEVICE_TYPE_VIDEOTOOLBOX. + * + * This API currently does not support frame allocation, as the raw VideoToolbox + * API does allocation, and FFmpeg itself never has the need to allocate frames. + * + * If the API user sets a custom pool, AVHWFramesContext.pool must return + * AVBufferRefs whose data pointer is a CVImageBufferRef or CVPixelBufferRef. + * + * Currently AVHWDeviceContext.hwctx and AVHWFramesContext.hwctx are always + * NULL. + */ + +/** + * Convert a VideoToolbox (actually CoreVideo) format to AVPixelFormat. + * Returns AV_PIX_FMT_NONE if no known equivalent was found. + */ +enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt); + +/** + * Convert an AVPixelFormat to a VideoToolbox (actually CoreVideo) format. + * Returns 0 if no known equivalent was found. + */ +uint32_t av_map_videotoolbox_format_from_pixfmt(enum AVPixelFormat pix_fmt); + +#endif /* AVUTIL_HWCONTEXT_VT_H */ diff --git a/libavutil/version.h b/libavutil/version.h index 6762bf300a..fb61dcc666 100644 --- a/libavutil/version.h +++ b/libavutil/version.h @@ -80,7 +80,7 @@ #define LIBAVUTIL_VERSION_MAJOR 55 -#define LIBAVUTIL_VERSION_MINOR 62 +#define LIBAVUTIL_VERSION_MINOR 63 #define LIBAVUTIL_VERSION_MICRO 100 #define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \ -- cgit v1.2.3