Merge commit '65e78a2e4b111627c0ebdf2c9baec95e5e21560d'

* commit '65e78a2e4b111627c0ebdf2c9baec95e5e21560d':
  x11grab: Refactor pixel format parsing

Conflicts:
	libavdevice/x11grab.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 71 insertions, 75 deletions

diff --git a/libavdevice/x11grab.c b/libavdevice/x11grab.c
index 6a0eb8dd38..120f754b10 100644
--- a/libavdevice/x11grab.c
+++ b/libavdevice/x11grab.c
@@ -172,6 +172,63 @@ static int setup_shm(AVFormatContext *s, Display *dpy, XImage **image)
     return 0;
 }
 
+static int pixfmt_from_image(AVFormatContext *s, XImage *image, int *pix_fmt)
+{
+    av_log(s, AV_LOG_DEBUG,
+           "Image r 0x%.6lx g 0x%.6lx b 0x%.6lx and depth %i\n",
+           image->red_mask,
+           image->green_mask,
+           image->blue_mask,
+           image->bits_per_pixel);
+
+    switch (image->bits_per_pixel) {
+    case 8:
+        *pix_fmt =  AV_PIX_FMT_PAL8;
+        break;
+    case 16:
+        if (image->red_mask   == 0xf800 &&
+            image->green_mask == 0x07e0 &&
+            image->blue_mask  == 0x001f) {
+            *pix_fmt = AV_PIX_FMT_RGB565;
+        } else if (image->red_mask   == 0x7c00 &&
+                   image->green_mask == 0x03e0 &&
+                   image->blue_mask  == 0x001f) {
+            *pix_fmt = AV_PIX_FMT_RGB555;
+        }
+        break;
+    case 24:
+        if (image->red_mask   == 0xff0000 &&
+            image->green_mask == 0x00ff00 &&
+            image->blue_mask  == 0x0000ff) {
+            *pix_fmt = AV_PIX_FMT_BGR24;
+        } else if (image->red_mask   == 0x0000ff &&
+                   image->green_mask == 0x00ff00 &&
+                   image->blue_mask  == 0xff0000) {
+            *pix_fmt = AV_PIX_FMT_RGB24;
+        }
+        break;
+    case 32:
+        if (image->red_mask   == 0xff0000 &&
+            image->green_mask == 0x00ff00 &&
+            image->blue_mask  == 0x0000ff ) {
+            *pix_fmt = AV_PIX_FMT_0RGB32;
+        }
+        break;
+    default:
+        av_log(s, AV_LOG_ERROR,
+               "XImages with RGB mask 0x%.6lx 0x%.6lx 0x%.6lx and depth %i "
+               "are currently not supported.\n",
+               image->red_mask,
+               image->green_mask,
+               image->blue_mask,
+               image->bits_per_pixel);
+
+        return AVERROR_PATCHWELCOME;
+    }
+
+    return 0;
+}
+
 /**
  * Initialize the x11 grab device demuxer (public device demuxer API).
  *
@@ -187,7 +244,6 @@ static int x11grab_read_header(AVFormatContext *s1)
     X11GrabContext *x11grab = s1->priv_data;
     Display *dpy;
     AVStream *st = NULL;
-    enum AVPixelFormat input_pixfmt;
     XImage *image;
     int x_off = 0, y_off = 0, ret = 0, screen, use_shm = 0;
     char *dpyname, *offset;
@@ -267,10 +323,20 @@ static int x11grab_read_header(AVFormatContext *s1)
                           AllPlanes, ZPixmap);
     }
 
-    switch (image->bits_per_pixel) {
-    case 8:
-        av_log(s1, AV_LOG_DEBUG, "8 bit palette\n");
-        input_pixfmt = AV_PIX_FMT_PAL8;
+    x11grab->frame_size = x11grab->width * x11grab->height * image->bits_per_pixel / 8;
+    x11grab->dpy        = dpy;
+    x11grab->time_base  = av_inv_q(x11grab->framerate);
+    x11grab->time_frame = av_gettime() / av_q2d(x11grab->time_base);
+    x11grab->x_off      = x_off;
+    x11grab->y_off      = y_off;
+    x11grab->image      = image;
+    x11grab->use_shm    = use_shm;
+
+    ret = pixfmt_from_image(s1, image, &st->codec->pix_fmt);
+    if (ret < 0)
+        goto out;
+
+    if (st->codec->pix_fmt == AV_PIX_FMT_PAL8) {
         color_map = DefaultColormap(dpy, screen);
         for (i = 0; i < 256; ++i)
             color[i].pixel = i;
@@ -280,83 +346,13 @@ static int x11grab_read_header(AVFormatContext *s1)
                                   (color[i].green & 0xFF00)      |
                                   (color[i].blue  & 0xFF00) >> 8;
         x11grab->palette_changed = 1;
-        break;
-    case 16:
-        if (image->red_mask   == 0xf800 &&
-            image->green_mask == 0x07e0 &&
-            image->blue_mask  == 0x001f) {
-            av_log(s1, AV_LOG_DEBUG, "16 bit RGB565\n");
-            input_pixfmt = AV_PIX_FMT_RGB565;
-        } else if (image->red_mask   == 0x7c00 &&
-                   image->green_mask == 0x03e0 &&
-                   image->blue_mask  == 0x001f) {
-            av_log(s1, AV_LOG_DEBUG, "16 bit RGB555\n");
-            input_pixfmt = AV_PIX_FMT_RGB555;
-        } else {
-            av_log(s1, AV_LOG_ERROR,
-                   "RGB ordering at image depth %i not supported ... aborting\n",
-                   image->bits_per_pixel);
-            av_log(s1, AV_LOG_ERROR,
-                   "color masks: r 0x%.6lx g 0x%.6lx b 0x%.6lx\n",
-                   image->red_mask, image->green_mask, image->blue_mask);
-            ret = AVERROR_PATCHWELCOME;
-            goto out;
-        }
-        break;
-    case 24:
-        if (image->red_mask   == 0xff0000 &&
-            image->green_mask == 0x00ff00 &&
-            image->blue_mask  == 0x0000ff) {
-            input_pixfmt = AV_PIX_FMT_BGR24;
-        } else if (image->red_mask   == 0x0000ff &&
-                   image->green_mask == 0x00ff00 &&
-                   image->blue_mask  == 0xff0000) {
-            input_pixfmt = AV_PIX_FMT_RGB24;
-        } else {
-            av_log(s1, AV_LOG_ERROR,
-                   "rgb ordering at image depth %i not supported ... aborting\n",
-                   image->bits_per_pixel);
-            av_log(s1, AV_LOG_ERROR,
-                   "color masks: r 0x%.6lx g 0x%.6lx b 0x%.6lx\n",
-                   image->red_mask, image->green_mask, image->blue_mask);
-            ret = AVERROR_PATCHWELCOME;
-            goto out;
-        }
-        break;
-    case 32:
-        if (        image->red_mask   == 0xff0000 &&
-                    image->green_mask == 0x00ff00 &&
-                    image->blue_mask  == 0x0000ff ) {
-            input_pixfmt = AV_PIX_FMT_0RGB32;
-        } else {
-            av_log(s1, AV_LOG_ERROR,"rgb ordering at image depth %i not supported ... aborting\n", image->bits_per_pixel);
-            av_log(s1, AV_LOG_ERROR, "color masks: r 0x%.6lx g 0x%.6lx b 0x%.6lx\n", image->red_mask, image->green_mask, image->blue_mask);
-            ret = AVERROR_PATCHWELCOME;
-            goto out;
-        }
-        break;
-    default:
-        av_log(s1, AV_LOG_ERROR,
-               "image depth %i not supported ... aborting\n",
-               image->bits_per_pixel);
-        ret = AVERROR_PATCHWELCOME;
-        goto out;
     }
 
-    x11grab->frame_size = x11grab->width * x11grab->height * image->bits_per_pixel / 8;
-    x11grab->dpy        = dpy;
-    x11grab->time_base  = av_inv_q(x11grab->framerate);
-    x11grab->time_frame = av_gettime() / av_q2d(x11grab->time_base);
-    x11grab->x_off      = x_off;
-    x11grab->y_off      = y_off;
-    x11grab->image      = image;
-    x11grab->use_shm    = use_shm;
 
     st->codec->codec_type = AVMEDIA_TYPE_VIDEO;
     st->codec->codec_id   = AV_CODEC_ID_RAWVIDEO;
     st->codec->width      = x11grab->width;
     st->codec->height     = x11grab->height;
-    st->codec->pix_fmt    = input_pixfmt;
     st->codec->time_base  = x11grab->time_base;
     st->codec->bit_rate   = x11grab->frame_size * 1 / av_q2d(x11grab->time_base) * 8;