hwcontext_vaapi: Improve format mapping

Give the entries in the VAAPI format map table an explicit type and add
functions to do the necessary lookups.  Add another field to this table
indicating whether the chroma planes are swapped (as in YV12), and use
that rather than explicit comparisons where swapping is needed.

1 files changed, 76 insertions, 57 deletions

diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 4088a96be4..8624369bb9 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -87,57 +87,82 @@ typedef struct VAAPIMapping {
     int flags;
 } VAAPIMapping;
 
-#define MAP(va, rt, av) { \
+typedef struct VAAPIFormat {
+    unsigned int fourcc;
+    unsigned int rt_format;
+    enum AVPixelFormat pix_fmt;
+    int chroma_planes_swapped;
+} VAAPIFormatDescriptor;
+
+#define MAP(va, rt, av, swap_uv) { \
         VA_FOURCC_ ## va, \
         VA_RT_FORMAT_ ## rt, \
-        AV_PIX_FMT_ ## av \
+        AV_PIX_FMT_ ## av, \
+        swap_uv, \
     }
 // The map fourcc <-> pix_fmt isn't bijective because of the annoying U/V
 // plane swap cases.  The frame handling below tries to hide these.
-static const struct {
-    unsigned int fourcc;
-    unsigned int rt_format;
-    enum AVPixelFormat pix_fmt;
-} vaapi_format_map[] = {
-    MAP(NV12, YUV420,  NV12),
-    MAP(YV12, YUV420,  YUV420P), // With U/V planes swapped.
-    MAP(IYUV, YUV420,  YUV420P),
+static const VAAPIFormatDescriptor vaapi_format_map[] = {
+    MAP(NV12, YUV420,  NV12,    0),
 #ifdef VA_FOURCC_I420
-    MAP(I420, YUV420,  YUV420P),
+    MAP(I420, YUV420,  YUV420P, 0),
 #endif
+    MAP(YV12, YUV420,  YUV420P, 1),
+    MAP(IYUV, YUV420,  YUV420P, 0),
+    MAP(422H, YUV422,  YUV422P, 0),
 #ifdef VA_FOURCC_YV16
-    MAP(YV16, YUV422,  YUV422P), // With U/V planes swapped.
+    MAP(YV16, YUV422,  YUV422P, 1),
 #endif
-    MAP(422H, YUV422,  YUV422P),
-    MAP(UYVY, YUV422,  UYVY422),
-    MAP(YUY2, YUV422,  YUYV422),
-    MAP(411P, YUV411,  YUV411P),
-    MAP(422V, YUV422,  YUV440P),
-    MAP(444P, YUV444,  YUV444P),
-    MAP(Y800, YUV400,  GRAY8),
+    MAP(UYVY, YUV422,  UYVY422, 0),
+    MAP(YUY2, YUV422,  YUYV422, 0),
+    MAP(411P, YUV411,  YUV411P, 0),
+    MAP(422V, YUV422,  YUV440P, 0),
+    MAP(444P, YUV444,  YUV444P, 0),
+    MAP(Y800, YUV400,  GRAY8,   0),
 #ifdef VA_FOURCC_P010
-    MAP(P010, YUV420_10BPP, P010),
+    MAP(P010, YUV420_10BPP, P010, 0),
 #endif
-    MAP(BGRA, RGB32,   BGRA),
-    MAP(BGRX, RGB32,   BGR0),
-    MAP(RGBA, RGB32,   RGBA),
-    MAP(RGBX, RGB32,   RGB0),
+    MAP(BGRA, RGB32,   BGRA, 0),
+    MAP(BGRX, RGB32,   BGR0, 0),
+    MAP(RGBA, RGB32,   RGBA, 0),
+    MAP(RGBX, RGB32,   RGB0, 0),
 #ifdef VA_FOURCC_ABGR
-    MAP(ABGR, RGB32,   ABGR),
-    MAP(XBGR, RGB32,   0BGR),
+    MAP(ABGR, RGB32,   ABGR, 0),
+    MAP(XBGR, RGB32,   0BGR, 0),
 #endif
-    MAP(ARGB, RGB32,   ARGB),
-    MAP(XRGB, RGB32,   0RGB),
+    MAP(ARGB, RGB32,   ARGB, 0),
+    MAP(XRGB, RGB32,   0RGB, 0),
 };
 #undef MAP
 
-static enum AVPixelFormat vaapi_pix_fmt_from_fourcc(unsigned int fourcc)
+static const VAAPIFormatDescriptor *
+    vaapi_format_from_fourcc(unsigned int fourcc)
 {
     int i;
     for (i = 0; i < FF_ARRAY_ELEMS(vaapi_format_map); i++)
         if (vaapi_format_map[i].fourcc == fourcc)
-            return vaapi_format_map[i].pix_fmt;
-    return AV_PIX_FMT_NONE;
+            return &vaapi_format_map[i];
+    return NULL;
+}
+
+static const VAAPIFormatDescriptor *
+    vaapi_format_from_pix_fmt(enum AVPixelFormat pix_fmt)
+{
+    int i;
+    for (i = 0; i < FF_ARRAY_ELEMS(vaapi_format_map); i++)
+        if (vaapi_format_map[i].pix_fmt == pix_fmt)
+            return &vaapi_format_map[i];
+    return NULL;
+}
+
+static enum AVPixelFormat vaapi_pix_fmt_from_fourcc(unsigned int fourcc)
+{
+    const VAAPIFormatDescriptor *desc;
+    desc = vaapi_format_from_fourcc(fourcc);
+    if (desc)
+        return desc->pix_fmt;
+    else
+        return AV_PIX_FMT_NONE;
 }
 
 static int vaapi_get_image_format(AVHWDeviceContext *hwdev,
@@ -461,22 +486,16 @@ static int vaapi_frames_init(AVHWFramesContext *hwfc)
     AVVAAPIFramesContext  *avfc = hwfc->hwctx;
     VAAPIFramesContext     *ctx = hwfc->internal->priv;
     AVVAAPIDeviceContext *hwctx = hwfc->device_ctx->hwctx;
+    const VAAPIFormatDescriptor *desc;
     VAImageFormat *expected_format;
     AVBufferRef *test_surface = NULL;
     VASurfaceID test_surface_id;
     VAImage test_image;
     VAStatus vas;
     int err, i;
-    unsigned int fourcc, rt_format;
 
-    for (i = 0; i < FF_ARRAY_ELEMS(vaapi_format_map); i++) {
-        if (vaapi_format_map[i].pix_fmt == hwfc->sw_format) {
-            fourcc    = vaapi_format_map[i].fourcc;
-            rt_format = vaapi_format_map[i].rt_format;
-            break;
-        }
-    }
-    if (i >= FF_ARRAY_ELEMS(vaapi_format_map)) {
+    desc = vaapi_format_from_pix_fmt(hwfc->sw_format);
+    if (!desc) {
         av_log(hwfc, AV_LOG_ERROR, "Unsupported format: %s.\n",
                av_get_pix_fmt_name(hwfc->sw_format));
         return AVERROR(EINVAL);
@@ -517,7 +536,7 @@ static int vaapi_frames_init(AVHWFramesContext *hwfc)
                     .type          = VASurfaceAttribPixelFormat,
                     .flags         = VA_SURFACE_ATTRIB_SETTABLE,
                     .value.type    = VAGenericValueTypeInteger,
-                    .value.value.i = fourcc,
+                    .value.value.i = desc->fourcc,
                 };
             }
             av_assert0(i == ctx->nb_attributes);
@@ -526,7 +545,7 @@ static int vaapi_frames_init(AVHWFramesContext *hwfc)
             ctx->nb_attributes = 0;
         }
 
-        ctx->rt_format = rt_format;
+        ctx->rt_format = desc->rt_format;
 
         if (hwfc->initial_pool_size > 0) {
             // This pool will be usable as a render target, so we need to store
@@ -716,6 +735,7 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
     AVVAAPIDeviceContext *hwctx = hwfc->device_ctx->hwctx;
     VAAPIFramesContext *ctx = hwfc->internal->priv;
     VASurfaceID surface_id;
+    const VAAPIFormatDescriptor *desc;
     VAImageFormat *image_format;
     VAAPIMapping *map;
     VAStatus vas;
@@ -824,11 +844,9 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
         dst->data[i] = (uint8_t*)address + map->image.offsets[i];
         dst->linesize[i] = map->image.pitches[i];
     }
-    if (
-#ifdef VA_FOURCC_YV16
-        map->image.format.fourcc == VA_FOURCC_YV16 ||
-#endif
-        map->image.format.fourcc == VA_FOURCC_YV12) {
+
+    desc = vaapi_format_from_fourcc(map->image.format.fourcc);
+    if (desc && desc->chroma_planes_swapped) {
         // Chroma planes are YVU rather than YUV, so swap them.
         FFSWAP(uint8_t*, dst->data[1], dst->data[2]);
     }
@@ -981,9 +999,10 @@ static int vaapi_map_from_drm(AVHWFramesContext *src_fc, AVFrame *dst,
         (AVHWFramesContext*)dst->hw_frames_ctx->data;
     AVVAAPIDeviceContext  *dst_dev = dst_fc->device_ctx->hwctx;
     const AVDRMFrameDescriptor *desc;
+    const VAAPIFormatDescriptor *format_desc;
     VASurfaceID surface_id;
     VAStatus vas;
-    uint32_t va_fourcc, va_rt_format;
+    uint32_t va_fourcc;
     int err, i, j, k;
 
     unsigned long buffer_handle;
@@ -1034,14 +1053,8 @@ static int vaapi_map_from_drm(AVHWFramesContext *src_fc, AVFrame *dst,
     av_log(dst_fc, AV_LOG_DEBUG, "Map DRM object %d to VAAPI as "
            "%08x.\n", desc->objects[0].fd, va_fourcc);
 
-    for (i = 0; i < FF_ARRAY_ELEMS(vaapi_format_map); i++) {
-        if (vaapi_format_map[i].fourcc == va_fourcc) {
-            va_rt_format = vaapi_format_map[i].rt_format;
-            break;
-        }
-    }
-
-    av_assert0(i < FF_ARRAY_ELEMS(vaapi_format_map));
+    format_desc = vaapi_format_from_fourcc(va_fourcc);
+    av_assert0(format_desc);
 
     buffer_handle = desc->objects[0].fd;
     buffer_desc.pixel_format = va_fourcc;
@@ -1062,7 +1075,13 @@ static int vaapi_map_from_drm(AVHWFramesContext *src_fc, AVFrame *dst,
     }
     buffer_desc.num_planes = k;
 
-    vas = vaCreateSurfaces(dst_dev->display, va_rt_format,
+    if (format_desc->chroma_planes_swapped &&
+        buffer_desc.num_planes == 3) {
+        FFSWAP(uint32_t, buffer_desc.pitches[1], buffer_desc.pitches[2]);
+        FFSWAP(uint32_t, buffer_desc.offsets[1], buffer_desc.offsets[2]);
+    }
+
+    vas = vaCreateSurfaces(dst_dev->display, format_desc->rt_format,
                            src->width, src->height,
                            &surface_id, 1,
                            attrs, FF_ARRAY_ELEMS(attrs));