lavu/hwcontext_vaapi: Add Windows/VAAPI support with vaGetDisplayWin32

Libva 2.17+ adds a new libva-win32 node and Mesa 22.3 adds a VAAPI driver
based on Direct3D 12 for Windows. Both of them are available at:
https://www.nuget.org/packages/Microsoft.Direct3D.VideoAccelerationCompatibilityPack

Initial review at https://github.com/intel-media-ci/ffmpeg/pull/619/

Signed-off-by: Sil Vilerino <sivileri@microsoft.com>
Reviewed-by: Dmitry Rogozhkin <dmitry.v.rogozhkin@intel.com>
Reviewed-by: Wu, Tong1 <tong1.wu@intel.com>

1 files changed, 76 insertions, 2 deletions

diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 90c2c191d9..6c3a227ddd 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -18,6 +18,15 @@
 
 #include "config.h"
 
+#if HAVE_VAAPI_WIN32
+#   include <windows.h>
+#define COBJMACROS
+#   include <initguid.h>
+#   include <dxgi1_2.h>
+#   include "compat/w32dlfcn.h"
+#   include <va/va_win32.h>
+typedef HRESULT (WINAPI *PFN_CREATE_DXGI_FACTORY)(REFIID riid, void **ppFactory);
+#endif
 #if HAVE_VAAPI_X11
 #   include <va/va_x11.h>
 #endif
@@ -1663,7 +1672,7 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
     VAAPIDevicePriv *priv;
     VADisplay display = NULL;
     const AVDictionaryEntry *ent;
-    int try_drm, try_x11, try_all;
+    int try_drm, try_x11, try_win32, try_all;
 
     priv = av_mallocz(sizeof(*priv));
     if (!priv)
@@ -1676,11 +1685,13 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
 
     ent = av_dict_get(opts, "connection_type", NULL, 0);
     if (ent) {
-        try_all = try_drm = try_x11 = 0;
+        try_all = try_drm = try_x11 = try_win32 = 0;
         if (!strcmp(ent->value, "drm")) {
             try_drm = 1;
         } else if (!strcmp(ent->value, "x11")) {
             try_x11 = 1;
+        } else if (!strcmp(ent->value, "win32")) {
+            try_win32 = 1;
         } else {
             av_log(ctx, AV_LOG_ERROR, "Invalid connection type %s.\n",
                    ent->value);
@@ -1690,6 +1701,7 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
         try_all = 1;
         try_drm = HAVE_VAAPI_DRM;
         try_x11 = HAVE_VAAPI_X11;
+        try_win32 = HAVE_VAAPI_WIN32;
     }
 
 #if HAVE_VAAPI_DRM
@@ -1797,6 +1809,68 @@ static int vaapi_device_create(AVHWDeviceContext *ctx, const char *device,
     }
 #endif
 
+#if HAVE_VAAPI_WIN32
+    if (!display && try_win32) {
+        // Try to create a display from the specified device, if any.
+        if (!device) {
+            display = vaGetDisplayWin32(NULL);
+        } else {
+            IDXGIFactory2 *pDXGIFactory = NULL;
+            IDXGIAdapter *pAdapter = NULL;
+#if !HAVE_UWP
+            HANDLE dxgi = dlopen("dxgi.dll", 0);
+            if (!dxgi) {
+                av_log(ctx, AV_LOG_ERROR, "Failed to load dxgi.dll\n");
+                return AVERROR_UNKNOWN;
+            }
+            PFN_CREATE_DXGI_FACTORY pfnCreateDXGIFactory =
+                (PFN_CREATE_DXGI_FACTORY)dlsym(dxgi, "CreateDXGIFactory");
+            if (!pfnCreateDXGIFactory) {
+                av_log(ctx, AV_LOG_ERROR, "CreateDXGIFactory load failed\n");
+                dlclose(dxgi);
+                return AVERROR_UNKNOWN;
+            }
+#else
+            // In UWP (which lacks LoadLibrary), CreateDXGIFactory isn't
+            // available, only CreateDXGIFactory1
+            PFN_CREATE_DXGI_FACTORY pfnCreateDXGIFactory =
+                (PFN_CREATE_DXGI_FACTORY)CreateDXGIFactory1;
+#endif
+            if (SUCCEEDED(pfnCreateDXGIFactory(&IID_IDXGIFactory2,
+                                              (void **)&pDXGIFactory))) {
+                int adapter = atoi(device);
+                if (SUCCEEDED(IDXGIFactory2_EnumAdapters(pDXGIFactory,
+                                                         adapter,
+                                                         &pAdapter))) {
+                    DXGI_ADAPTER_DESC desc;
+                    if (SUCCEEDED(IDXGIAdapter2_GetDesc(pAdapter, &desc))) {
+                        av_log(ctx, AV_LOG_INFO,
+                              "Using device %04x:%04x (%ls) - LUID %lu %ld.\n",
+                              desc.VendorId, desc.DeviceId, desc.Description,
+                              desc.AdapterLuid.LowPart,
+                              desc.AdapterLuid.HighPart);
+                        display = vaGetDisplayWin32(&desc.AdapterLuid);
+                    }
+                    IDXGIAdapter_Release(pAdapter);
+                }
+                IDXGIFactory2_Release(pDXGIFactory);
+            }
+#if !HAVE_UWP
+            dlclose(dxgi);
+#endif
+        }
+
+        if (!display) {
+            av_log(ctx, AV_LOG_ERROR, "Cannot open a VA display "
+                    "from Win32 display.\n");
+            return AVERROR_UNKNOWN;
+        }
+
+        av_log(ctx, AV_LOG_VERBOSE, "Opened VA display via "
+                "Win32 display.\n");
+    }
+#endif
+
     if (!display) {
         if (device)
             av_log(ctx, AV_LOG_ERROR, "No VA display found for "