hwcontext_vaapi: Try to support the VDPAU wrapper

The driver is somewhat bitrotten (not updated for years) but is still
usable for decoding with this change.  To support it, this adds a new
driver quirk to indicate no support at all for surface attributes.

Based on a patch by wm4 <nfxjfg@googlemail.com>.

2 files changed, 52 insertions, 34 deletions

diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index f8719aaf0a..23bce76605 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -153,7 +153,8 @@ static int vaapi_frames_get_constraints(AVHWDeviceContext *hwdev,
     unsigned int fourcc;
     int err, i, j, attr_count, pix_fmt_count;
 
-    if (config) {
+    if (config &&
+        !(hwctx->driver_quirks & AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES)) {
         attr_count = 0;
         vas = vaQuerySurfaceAttributes(hwctx->display, config->config_id,
                                        0, &attr_count);
@@ -271,6 +272,11 @@ static const struct {
         "ubit",
         AV_VAAPI_DRIVER_QUIRK_ATTRIB_MEMTYPE,
     },
+    {
+        "VDPAU wrapper",
+        "Splitted-Desktop Systems VDPAU backend for VA-API",
+        AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES,
+    },
 };
 
 static int vaapi_device_init(AVHWDeviceContext *hwdev)
@@ -449,43 +455,48 @@ static int vaapi_frames_init(AVHWFramesContext *hwfc)
     }
 
     if (!hwfc->pool) {
-        int need_memory_type = !(hwctx->driver_quirks & AV_VAAPI_DRIVER_QUIRK_ATTRIB_MEMTYPE);
-        int need_pixel_format = 1;
-        for (i = 0; i < avfc->nb_attributes; i++) {
-            if (ctx->attributes[i].type == VASurfaceAttribMemoryType)
-                need_memory_type  = 0;
-            if (ctx->attributes[i].type == VASurfaceAttribPixelFormat)
-                need_pixel_format = 0;
-        }
-        ctx->nb_attributes =
-            avfc->nb_attributes + need_memory_type + need_pixel_format;
+        if (!(hwctx->driver_quirks & AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES)) {
+            int need_memory_type = !(hwctx->driver_quirks & AV_VAAPI_DRIVER_QUIRK_ATTRIB_MEMTYPE);
+            int need_pixel_format = 1;
+            for (i = 0; i < avfc->nb_attributes; i++) {
+                if (ctx->attributes[i].type == VASurfaceAttribMemoryType)
+                    need_memory_type  = 0;
+                if (ctx->attributes[i].type == VASurfaceAttribPixelFormat)
+                    need_pixel_format = 0;
+            }
+            ctx->nb_attributes =
+                avfc->nb_attributes + need_memory_type + need_pixel_format;
 
-        ctx->attributes = av_malloc(ctx->nb_attributes *
+            ctx->attributes = av_malloc(ctx->nb_attributes *
                                         sizeof(*ctx->attributes));
-        if (!ctx->attributes) {
-            err = AVERROR(ENOMEM);
-            goto fail;
-        }
+            if (!ctx->attributes) {
+                err = AVERROR(ENOMEM);
+                goto fail;
+            }
 
-        for (i = 0; i < avfc->nb_attributes; i++)
-            ctx->attributes[i] = avfc->attributes[i];
-        if (need_memory_type) {
-            ctx->attributes[i++] = (VASurfaceAttrib) {
-                .type          = VASurfaceAttribMemoryType,
-                .flags         = VA_SURFACE_ATTRIB_SETTABLE,
-                .value.type    = VAGenericValueTypeInteger,
-                .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_VA,
-            };
-        }
-        if (need_pixel_format) {
-            ctx->attributes[i++] = (VASurfaceAttrib) {
-                .type          = VASurfaceAttribPixelFormat,
-                .flags         = VA_SURFACE_ATTRIB_SETTABLE,
-                .value.type    = VAGenericValueTypeInteger,
-                .value.value.i = fourcc,
-            };
+            for (i = 0; i < avfc->nb_attributes; i++)
+                ctx->attributes[i] = avfc->attributes[i];
+            if (need_memory_type) {
+                ctx->attributes[i++] = (VASurfaceAttrib) {
+                    .type          = VASurfaceAttribMemoryType,
+                    .flags         = VA_SURFACE_ATTRIB_SETTABLE,
+                    .value.type    = VAGenericValueTypeInteger,
+                    .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_VA,
+                };
+            }
+            if (need_pixel_format) {
+                ctx->attributes[i++] = (VASurfaceAttrib) {
+                    .type          = VASurfaceAttribPixelFormat,
+                    .flags         = VA_SURFACE_ATTRIB_SETTABLE,
+                    .value.type    = VAGenericValueTypeInteger,
+                    .value.value.i = fourcc,
+                };
+            }
+            av_assert0(i == ctx->nb_attributes);
+        } else {
+            ctx->attributes = NULL;
+            ctx->nb_attributes = 0;
         }
-        av_assert0(i == ctx->nb_attributes);
 
         ctx->rt_format = rt_format;
 
diff --git a/libavutil/hwcontext_vaapi.h b/libavutil/hwcontext_vaapi.h
index 9f1e78a414..529d26e07a 100644
--- a/libavutil/hwcontext_vaapi.h
+++ b/libavutil/hwcontext_vaapi.h
@@ -51,6 +51,13 @@ enum {
      * so the surface allocation code will not try to use it.
      */
     AV_VAAPI_DRIVER_QUIRK_ATTRIB_MEMTYPE = (1 << 2),
+
+    /**
+     * The driver does not support surface attributes at all.
+     * The surface allocation code will never pass them to surface allocation,
+     * and the results of the vaQuerySurfaceAttributes() call will be faked.
+     */
+    AV_VAAPI_DRIVER_QUIRK_SURFACE_ATTRIBUTES = (1 << 3),
 };
 
 /**