x11grab: Refactor pixel format parsing

1 files changed, 57 insertions, 60 deletions

diff --git a/libavdevice/x11grab.c b/libavdevice/x11grab.c
index 4ca5434a9e..03fd567db8 100644
--- a/libavdevice/x11grab.c
+++ b/libavdevice/x11grab.c
@@ -168,6 +168,59 @@ static int setup_shm(AVFormatContext *s, Display *dpy, XImage **image)
     return 0;
 }
 
+static int pixfmt_from_image(AVFormatContext *s, XImage *image, int *pix_fmt)
+{
+    av_log(s, AV_LOG_DEBUG,
+           "Image r 0x%.6lx g 0x%.6lx b 0x%.6lx and depth %i\n",
+           image->red_mask,
+           image->green_mask,
+           image->blue_mask,
+           image->bits_per_pixel);
+
+    switch (image->bits_per_pixel) {
+    case 8:
+        *pix_fmt =  AV_PIX_FMT_PAL8;
+        break;
+    case 16:
+        if (image->red_mask   == 0xf800 &&
+            image->green_mask == 0x07e0 &&
+            image->blue_mask  == 0x001f) {
+            *pix_fmt = AV_PIX_FMT_RGB565;
+        } else if (image->red_mask   == 0x7c00 &&
+                   image->green_mask == 0x03e0 &&
+                   image->blue_mask  == 0x001f) {
+            *pix_fmt = AV_PIX_FMT_RGB555;
+        }
+        break;
+    case 24:
+        if (image->red_mask   == 0xff0000 &&
+            image->green_mask == 0x00ff00 &&
+            image->blue_mask  == 0x0000ff) {
+            *pix_fmt = AV_PIX_FMT_BGR24;
+        } else if (image->red_mask   == 0x0000ff &&
+                   image->green_mask == 0x00ff00 &&
+                   image->blue_mask  == 0xff0000) {
+            *pix_fmt = AV_PIX_FMT_RGB24;
+        }
+        break;
+    case 32:
+        *pix_fmt = AV_PIX_FMT_RGB32;
+        break;
+    default:
+        av_log(s, AV_LOG_ERROR,
+               "XImages with RGB mask 0x%.6lx 0x%.6lx 0x%.6lx and depth %i "
+               "are currently not supported.\n",
+               image->red_mask,
+               image->green_mask,
+               image->blue_mask,
+               image->bits_per_pixel);
+
+        return AVERROR_PATCHWELCOME;
+    }
+
+    return 0;
+}
+
 /**
  * Initialize the x11 grab device demuxer (public device demuxer API).
  *
@@ -183,7 +236,6 @@ static int x11grab_read_header(AVFormatContext *s1)
     X11GrabContext *x11grab = s1->priv_data;
     Display *dpy;
     AVStream *st = NULL;
-    enum AVPixelFormat input_pixfmt;
     XImage *image;
     int x_off = 0, y_off = 0, ret = 0, screen, use_shm;
     char *param, *offset;
@@ -266,64 +318,6 @@ static int x11grab_read_header(AVFormatContext *s1)
                           AllPlanes, ZPixmap);
     }
 
-    switch (image->bits_per_pixel) {
-    case 8:
-        av_log(s1, AV_LOG_DEBUG, "8 bit palette\n");
-        input_pixfmt = AV_PIX_FMT_PAL8;
-        break;
-    case 16:
-        if (image->red_mask   == 0xf800 &&
-            image->green_mask == 0x07e0 &&
-            image->blue_mask  == 0x001f) {
-            av_log(s1, AV_LOG_DEBUG, "16 bit RGB565\n");
-            input_pixfmt = AV_PIX_FMT_RGB565;
-        } else if (image->red_mask   == 0x7c00 &&
-                   image->green_mask == 0x03e0 &&
-                   image->blue_mask  == 0x001f) {
-            av_log(s1, AV_LOG_DEBUG, "16 bit RGB555\n");
-            input_pixfmt = AV_PIX_FMT_RGB555;
-        } else {
-            av_log(s1, AV_LOG_ERROR,
-                   "RGB ordering at image depth %i not supported ... aborting\n",
-                   image->bits_per_pixel);
-            av_log(s1, AV_LOG_ERROR,
-                   "color masks: r 0x%.6lx g 0x%.6lx b 0x%.6lx\n",
-                   image->red_mask, image->green_mask, image->blue_mask);
-            ret = AVERROR(EIO);
-            goto out;
-        }
-        break;
-    case 24:
-        if (image->red_mask   == 0xff0000 &&
-            image->green_mask == 0x00ff00 &&
-            image->blue_mask  == 0x0000ff) {
-            input_pixfmt = AV_PIX_FMT_BGR24;
-        } else if (image->red_mask   == 0x0000ff &&
-                   image->green_mask == 0x00ff00 &&
-                   image->blue_mask  == 0xff0000) {
-            input_pixfmt = AV_PIX_FMT_RGB24;
-        } else {
-            av_log(s1, AV_LOG_ERROR,
-                   "rgb ordering at image depth %i not supported ... aborting\n",
-                   image->bits_per_pixel);
-            av_log(s1, AV_LOG_ERROR,
-                   "color masks: r 0x%.6lx g 0x%.6lx b 0x%.6lx\n",
-                   image->red_mask, image->green_mask, image->blue_mask);
-            ret = AVERROR(EIO);
-            goto out;
-        }
-        break;
-    case 32:
-        input_pixfmt = AV_PIX_FMT_RGB32;
-        break;
-    default:
-        av_log(s1, AV_LOG_ERROR,
-               "image depth %i not supported ... aborting\n",
-               image->bits_per_pixel);
-        ret = AVERROR(EINVAL);
-        goto out;
-    }
-
     x11grab->frame_size = x11grab->width * x11grab->height * image->bits_per_pixel / 8;
     x11grab->dpy        = dpy;
     x11grab->time_base  = (AVRational) { framerate.den, framerate.num };
@@ -333,11 +327,14 @@ static int x11grab_read_header(AVFormatContext *s1)
     x11grab->image      = image;
     x11grab->use_shm    = use_shm;
 
+    ret = pixfmt_from_image(s1, image, &st->codec->pix_fmt);
+    if (ret < 0)
+        goto out;
+
     st->codec->codec_type = AVMEDIA_TYPE_VIDEO;
     st->codec->codec_id   = AV_CODEC_ID_RAWVIDEO;
     st->codec->width      = x11grab->width;
     st->codec->height     = x11grab->height;
-    st->codec->pix_fmt    = input_pixfmt;
     st->codec->time_base  = x11grab->time_base;
     st->codec->bit_rate   = x11grab->frame_size * 1 / av_q2d(x11grab->time_base) * 8;