swscale: fix YUV420P 9/10bit support.

Fix handling of input if not in native endianness, and add support for
9/10-bit output. This allows us to force endianness of YUV420P 9/10bit
in the H264/10bit fate tests, which should fix them on big-endian
systems.

1 files changed, 22 insertions, 18 deletions

diff --git a/libswscale/swscale_template.c b/libswscale/swscale_template.c
index 81a8d66277..fe872561cb 100644
--- a/libswscale/swscale_template.c
+++ b/libswscale/swscale_template.c
@@ -164,6 +164,8 @@ static inline void LEToUV_c(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1,
                             const uint8_t *src2, long width, uint32_t *unused)
 {
     int i;
+    // FIXME I don't think this code is right for YUV444/422, since then h is not subsampled so
+    // we need to skip each second pixel. Same for BEToUV.
     for (i=0; i<width; i++) {
         dstU[i]= src1[2*i + 1];
         dstV[i]= src2[2*i + 1];
@@ -226,8 +228,8 @@ static inline void nv21ToUV_c(uint8_t *dstU, uint8_t *dstV,
 }
 
 // FIXME Maybe dither instead.
-#define YUV_NBPS(depth) \
-static inline void yuv ## depth ## ToUV_c(uint8_t *dstU, uint8_t *dstV, \
+#define YUV_NBPS(depth, endianness, rfunc) \
+static inline void endianness ## depth ## ToUV_c(uint8_t *dstU, uint8_t *dstV, \
                                           const uint8_t *_srcU, const uint8_t *_srcV, \
                                           long width, uint32_t *unused) \
 { \
@@ -235,21 +237,23 @@ static inline void yuv ## depth ## ToUV_c(uint8_t *dstU, uint8_t *dstV, \
     const uint16_t *srcU = (const uint16_t*)_srcU; \
     const uint16_t *srcV = (const uint16_t*)_srcV; \
     for (i = 0; i < width; i++) { \
-        dstU[i] = srcU[i]>>(depth-8); \
-        dstV[i] = srcV[i]>>(depth-8); \
+        dstU[i] = rfunc(&srcU[i])>>(depth-8); \
+        dstV[i] = rfunc(&srcV[i])>>(depth-8); \
     } \
 } \
 \
-static inline void yuv ## depth ## ToY_c(uint8_t *dstY, const uint8_t *_srcY, long width, uint32_t *unused) \
+static inline void endianness ## depth ## ToY_c(uint8_t *dstY, const uint8_t *_srcY, long width, uint32_t *unused) \
 { \
     int i; \
     const uint16_t *srcY = (const uint16_t*)_srcY; \
     for (i = 0; i < width; i++) \
-        dstY[i] = srcY[i]>>(depth-8); \
+        dstY[i] = rfunc(&srcY[i])>>(depth-8); \
 } \
 
-YUV_NBPS( 9)
-YUV_NBPS(10)
+YUV_NBPS( 9, LE, AV_RL16)
+YUV_NBPS( 9, BE, AV_RB16)
+YUV_NBPS(10, LE, AV_RL16)
+YUV_NBPS(10, BE, AV_RB16)
 
 static inline void bgr24ToY_c(uint8_t *dst, const uint8_t *src,
                               long width, uint32_t *unused)
@@ -666,7 +670,7 @@ static int swScale_c(SwsContext *c, const uint8_t* src[], int srcStride[],
             } else if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) { //YV12 like
                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
                 if ((dstY&chrSkipMask) || isGray(dstFormat)) uDest=vDest= NULL; //FIXME split functions in lumi / chromi
-                if (is16BPS(dstFormat)) {
+                if (is16BPS(dstFormat) || is9_OR_10BPS(dstFormat)) {
                     yuv2yuvX16inC(
                                   vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
                                   vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
@@ -743,7 +747,7 @@ static int swScale_c(SwsContext *c, const uint8_t* src[], int srcStride[],
             } else if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) { //YV12
                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
                 if ((dstY&chrSkipMask) || isGray(dstFormat)) uDest=vDest= NULL; //FIXME split functions in lumi / chromi
-                if (is16BPS(dstFormat)) {
+                if (is16BPS(dstFormat) || is9_OR_10BPS(dstFormat)) {
                     yuv2yuvX16inC(
                                   vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
                                   vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
@@ -816,10 +820,10 @@ static void sws_init_swScale_c(SwsContext *c)
         case PIX_FMT_PAL8     :
         case PIX_FMT_BGR4_BYTE:
         case PIX_FMT_RGB4_BYTE: c->chrToYV12 = palToUV; break;
-        case PIX_FMT_YUV420P9BE:
-        case PIX_FMT_YUV420P9LE: c->chrToYV12 = yuv9ToUV_c; break;
-        case PIX_FMT_YUV420P10BE:
-        case PIX_FMT_YUV420P10LE: c->chrToYV12 = yuv10ToUV_c; break;
+        case PIX_FMT_YUV420P9BE: c->chrToYV12 = BE9ToUV_c; break;
+        case PIX_FMT_YUV420P9LE: c->chrToYV12 = LE9ToUV_c; break;
+        case PIX_FMT_YUV420P10BE: c->chrToYV12 = BE10ToUV_c; break;
+        case PIX_FMT_YUV420P10LE: c->chrToYV12 = LE10ToUV_c; break;
         case PIX_FMT_YUV420P16BE:
         case PIX_FMT_YUV422P16BE:
         case PIX_FMT_YUV444P16BE: c->chrToYV12 = BEToUV_c; break;
@@ -866,10 +870,10 @@ static void sws_init_swScale_c(SwsContext *c)
     c->lumToYV12 = NULL;
     c->alpToYV12 = NULL;
     switch (srcFormat) {
-    case PIX_FMT_YUV420P9BE:
-    case PIX_FMT_YUV420P9LE: c->lumToYV12 = yuv9ToY_c; break;
-    case PIX_FMT_YUV420P10BE:
-    case PIX_FMT_YUV420P10LE: c->lumToYV12 = yuv10ToY_c; break;
+    case PIX_FMT_YUV420P9BE: c->lumToYV12 = BE9ToY_c; break;
+    case PIX_FMT_YUV420P9LE: c->lumToYV12 = LE9ToY_c; break;
+    case PIX_FMT_YUV420P10BE: c->lumToYV12 = BE10ToY_c; break;
+    case PIX_FMT_YUV420P10LE: c->lumToYV12 = LE10ToY_c; break;
     case PIX_FMT_YUYV422  :
     case PIX_FMT_YUV420P16BE:
     case PIX_FMT_YUV422P16BE: