hwcontext_vaapi: Frame mapping support

Can map to any supported software format (using a GPU copy if it
doesn't actually match the surface format underneath).

1 files changed, 52 insertions, 41 deletions

diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 9617979d7c..1988729637 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -70,20 +70,12 @@ typedef struct VAAPIFramesContext {
     int derive_works;
 } VAAPIFramesContext;
 
-enum {
-    VAAPI_MAP_READ   = 0x01,
-    VAAPI_MAP_WRITE  = 0x02,
-    VAAPI_MAP_DIRECT = 0x04,
-};
-
-typedef struct VAAPISurfaceMap {
-    // The source hardware frame of this mapping (with hw_frames_ctx set).
-    const AVFrame *source;
-    // VAAPI_MAP_* flags which apply to this mapping.
-    int flags;
+typedef struct VAAPIMapping {
     // Handle to the derived or copied image which is mapped.
     VAImage image;
-} VAAPISurfaceMap;
+    // The mapping flags actually used.
+    int flags;
+} VAAPIMapping;
 
 #define MAP(va, rt, av) { \
         VA_FOURCC_ ## va, \
@@ -140,7 +132,8 @@ static int vaapi_get_image_format(AVHWDeviceContext *hwdev,
 
     for (i = 0; i < ctx->nb_formats; i++) {
         if (ctx->formats[i].pix_fmt == pix_fmt) {
-            *image_format = &ctx->formats[i].image_format;
+            if (image_format)
+                *image_format = &ctx->formats[i].image_format;
             return 0;
         }
     }
@@ -630,17 +623,15 @@ static int vaapi_transfer_get_formats(AVHWFramesContext *hwfc,
     return 0;
 }
 
-static void vaapi_unmap_frame(void *opaque, uint8_t *data)
+static void vaapi_unmap_frame(AVHWFramesContext *hwfc,
+                              HWMapDescriptor *hwmap)
 {
-    AVHWFramesContext *hwfc = opaque;
     AVVAAPIDeviceContext *hwctx = hwfc->device_ctx->hwctx;
-    VAAPISurfaceMap *map = (VAAPISurfaceMap*)data;
-    const AVFrame *src;
+    VAAPIMapping           *map = hwmap->priv;
     VASurfaceID surface_id;
     VAStatus vas;
 
-    src = map->source;
-    surface_id = (VASurfaceID)(uintptr_t)src->data[3];
+    surface_id = (VASurfaceID)(uintptr_t)hwmap->source->data[3];
     av_log(hwfc, AV_LOG_DEBUG, "Unmap surface %#x.\n", surface_id);
 
     vas = vaUnmapBuffer(hwctx->display, map->image.buf);
@@ -649,8 +640,8 @@ static void vaapi_unmap_frame(void *opaque, uint8_t *data)
                "%#x: %d (%s).\n", surface_id, vas, vaErrorStr(vas));
     }
 
-    if ((map->flags & VAAPI_MAP_WRITE) &&
-        !(map->flags & VAAPI_MAP_DIRECT)) {
+    if ((map->flags & AV_HWFRAME_MAP_WRITE) &&
+        !(map->flags & AV_HWFRAME_MAP_DIRECT)) {
         vas = vaPutImage(hwctx->display, surface_id, map->image.image_id,
                          0, 0, hwfc->width, hwfc->height,
                          0, 0, hwfc->width, hwfc->height);
@@ -676,7 +667,7 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
     VAAPIFramesContext *ctx = hwfc->internal->priv;
     VASurfaceID surface_id;
     VAImageFormat *image_format;
-    VAAPISurfaceMap *map;
+    VAAPIMapping *map;
     VAStatus vas;
     void *address = NULL;
     int err, i;
@@ -684,13 +675,13 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
     surface_id = (VASurfaceID)(uintptr_t)src->data[3];
     av_log(hwfc, AV_LOG_DEBUG, "Map surface %#x.\n", surface_id);
 
-    if (!ctx->derive_works && (flags & VAAPI_MAP_DIRECT)) {
+    if (!ctx->derive_works && (flags & AV_HWFRAME_MAP_DIRECT)) {
         // Requested direct mapping but it is not possible.
         return AVERROR(EINVAL);
     }
     if (dst->format == AV_PIX_FMT_NONE)
         dst->format = hwfc->sw_format;
-    if (dst->format != hwfc->sw_format && (flags & VAAPI_MAP_DIRECT)) {
+    if (dst->format != hwfc->sw_format && (flags & AV_HWFRAME_MAP_DIRECT)) {
         // Requested direct mapping but the formats do not match.
         return AVERROR(EINVAL);
     }
@@ -701,12 +692,10 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
         return AVERROR(EINVAL);
     }
 
-    map = av_malloc(sizeof(VAAPISurfaceMap));
+    map = av_malloc(sizeof(*map));
     if (!map)
         return AVERROR(ENOMEM);
-
-    map->source         = src;
-    map->flags          = flags;
+    map->flags = flags;
     map->image.image_id = VA_INVALID_ID;
 
     vas = vaSyncSurface(hwctx->display, surface_id);
@@ -724,8 +713,8 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
     // faster with a copy routine which is aware of the limitation, but we
     // assume for now that the user is not aware of that and would therefore
     // prefer not to be given direct-mapped memory if they request read access.
-    if (ctx->derive_works &&
-        ((flags & VAAPI_MAP_DIRECT) || !(flags & VAAPI_MAP_READ))) {
+    if (ctx->derive_works && dst->format == hwfc->sw_format &&
+        ((flags & AV_HWFRAME_MAP_DIRECT) || !(flags & AV_HWFRAME_MAP_READ))) {
         vas = vaDeriveImage(hwctx->display, surface_id, &map->image);
         if (vas != VA_STATUS_SUCCESS) {
             av_log(hwfc, AV_LOG_ERROR, "Failed to derive image from "
@@ -741,7 +730,7 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
             err = AVERROR(EIO);
             goto fail;
         }
-        map->flags |= VAAPI_MAP_DIRECT;
+        map->flags |= AV_HWFRAME_MAP_DIRECT;
     } else {
         vas = vaCreateImage(hwctx->display, image_format,
                             hwfc->width, hwfc->height, &map->image);
@@ -752,7 +741,7 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
             err = AVERROR(EIO);
             goto fail;
         }
-        if (flags & VAAPI_MAP_READ) {
+        if (!(flags & AV_HWFRAME_MAP_OVERWRITE)) {
             vas = vaGetImage(hwctx->display, surface_id, 0, 0,
                              hwfc->width, hwfc->height, map->image.image_id);
             if (vas != VA_STATUS_SUCCESS) {
@@ -773,6 +762,11 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
         goto fail;
     }
 
+    err = ff_hwframe_map_create(src->hw_frames_ctx,
+                                dst, src, &vaapi_unmap_frame, map);
+    if (err < 0)
+        goto fail;
+
     dst->width  = src->width;
     dst->height = src->height;
 
@@ -789,13 +783,6 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
         FFSWAP(uint8_t*, dst->data[1], dst->data[2]);
     }
 
-    dst->buf[0] = av_buffer_create((uint8_t*)map, sizeof(*map),
-                                   &vaapi_unmap_frame, hwfc, 0);
-    if (!dst->buf[0]) {
-        err = AVERROR(ENOMEM);
-        goto fail;
-    }
-
     return 0;
 
 fail:
@@ -823,7 +810,7 @@ static int vaapi_transfer_data_from(AVHWFramesContext *hwfc,
         return AVERROR(ENOMEM);
     map->format = dst->format;
 
-    err = vaapi_map_frame(hwfc, map, src, VAAPI_MAP_READ);
+    err = vaapi_map_frame(hwfc, map, src, AV_HWFRAME_MAP_READ);
     if (err)
         goto fail;
 
@@ -854,7 +841,7 @@ static int vaapi_transfer_data_to(AVHWFramesContext *hwfc,
         return AVERROR(ENOMEM);
     map->format = src->format;
 
-    err = vaapi_map_frame(hwfc, map, dst, VAAPI_MAP_WRITE);
+    err = vaapi_map_frame(hwfc, map, dst, AV_HWFRAME_MAP_WRITE | AV_HWFRAME_MAP_OVERWRITE);
     if (err)
         goto fail;
 
@@ -871,6 +858,28 @@ fail:
     return err;
 }
 
+static int vaapi_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
+                          const AVFrame *src, int flags)
+{
+    int err;
+
+    if (dst->format != AV_PIX_FMT_NONE) {
+        err = vaapi_get_image_format(hwfc->device_ctx, dst->format, NULL);
+        if (err < 0)
+            return AVERROR(ENOSYS);
+    }
+
+    err = vaapi_map_frame(hwfc, dst, src, flags);
+    if (err)
+        return err;
+
+    err = av_frame_copy_props(dst, src);
+    if (err)
+        return err;
+
+    return 0;
+}
+
 static void vaapi_device_free(AVHWDeviceContext *ctx)
 {
     AVVAAPIDeviceContext *hwctx = ctx->hwctx;
@@ -993,6 +1002,8 @@ const HWContextType ff_hwcontext_type_vaapi = {
     .transfer_get_formats   = &vaapi_transfer_get_formats,
     .transfer_data_to       = &vaapi_transfer_data_to,
     .transfer_data_from     = &vaapi_transfer_data_from,
+    .map_to                 = NULL,
+    .map_from               = &vaapi_map_from,
 
     .pix_fmts = (const enum AVPixelFormat[]) {
         AV_PIX_FMT_VAAPI,