dxva: support DXGI_FORMAT_420_OPAQUE decoding

Some devices (some phones, apparently) will support only this opaque
format. Of course this won't work with CLI, because copying data
directly is not supported.

Automatic frame allocation (setting AVCodecContext.hw_device_ctx) does
not support this mode, even if it's the only supported mode. But since
opaque surfaces are generally less useful, that's probably ok.

Merges Libav commit 5030e3856c2126fb829edb828f5aae011d178eb4.

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

3 files changed, 25 insertions, 4 deletions

diff --git a/libavcodec/dxva2.c b/libavcodec/dxva2.c
index e9a6605b5f..76e8249bef 100644
--- a/libavcodec/dxva2.c
+++ b/libavcodec/dxva2.c
@@ -393,19 +393,28 @@ static int d3d11va_get_decoder_configuration(AVCodecContext *avctx,
     return ret;
 }
 
+static DXGI_FORMAT d3d11va_map_sw_to_hw_format(enum AVPixelFormat pix_fmt)
+{
+    switch (pix_fmt) {
+    case AV_PIX_FMT_NV12:       return DXGI_FORMAT_NV12;
+    case AV_PIX_FMT_P010:       return DXGI_FORMAT_P010;
+    case AV_PIX_FMT_YUV420P:    return DXGI_FORMAT_420_OPAQUE;
+    default:                    return DXGI_FORMAT_UNKNOWN;
+    }
+}
+
 static int d3d11va_create_decoder(AVCodecContext *avctx)
 {
     FFDXVASharedContext *sctx = DXVA_SHARED_CONTEXT(avctx);
     GUID *guid_list;
     unsigned guid_count, i;
     GUID decoder_guid;
-    DXGI_FORMAT surface_format = avctx->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ?
-                                 DXGI_FORMAT_P010 : DXGI_FORMAT_NV12;
     D3D11_VIDEO_DECODER_DESC desc = { 0 };
     D3D11_VIDEO_DECODER_CONFIG config;
     AVHWFramesContext *frames_ctx = (AVHWFramesContext *)avctx->hw_frames_ctx->data;
     AVD3D11VADeviceContext *device_hwctx = frames_ctx->device_ctx->hwctx;
     AVD3D11VAFramesContext *frames_hwctx = frames_ctx->hwctx;
+    DXGI_FORMAT surface_format = d3d11va_map_sw_to_hw_format(frames_ctx->sw_format);
     D3D11_TEXTURE2D_DESC texdesc;
     HRESULT hr;
     int ret;
diff --git a/libavutil/hwcontext_d3d11va.c b/libavutil/hwcontext_d3d11va.c
index 523a6d2dc6..376c76e5cf 100644
--- a/libavutil/hwcontext_d3d11va.c
+++ b/libavutil/hwcontext_d3d11va.c
@@ -83,8 +83,11 @@ static const struct {
     DXGI_FORMAT d3d_format;
     enum AVPixelFormat pix_fmt;
 } supported_formats[] = {
-    { DXGI_FORMAT_NV12, AV_PIX_FMT_NV12 },
-    { DXGI_FORMAT_P010, AV_PIX_FMT_P010 },
+    { DXGI_FORMAT_NV12,         AV_PIX_FMT_NV12 },
+    { DXGI_FORMAT_P010,         AV_PIX_FMT_P010 },
+    // Special opaque formats. The pix_fmt is merely a place holder, as the
+    // opaque format cannot be accessed directly.
+    { DXGI_FORMAT_420_OPAQUE,   AV_PIX_FMT_YUV420P },
 };
 
 static void d3d11va_default_lock(void *ctx)
@@ -270,6 +273,7 @@ static int d3d11va_transfer_get_formats(AVHWFramesContext *ctx,
                                         enum AVHWFrameTransferDirection dir,
                                         enum AVPixelFormat **formats)
 {
+    D3D11VAFramesContext *s = ctx->internal->priv;
     enum AVPixelFormat *fmts;
 
     fmts = av_malloc_array(2, sizeof(*fmts));
@@ -279,6 +283,10 @@ static int d3d11va_transfer_get_formats(AVHWFramesContext *ctx,
     fmts[0] = ctx->sw_format;
     fmts[1] = AV_PIX_FMT_NONE;
 
+    // Don't signal support for opaque formats. Actual access would fail.
+    if (s->format == DXGI_FORMAT_420_OPAQUE)
+        fmts[0] = AV_PIX_FMT_NONE;
+
     *formats = fmts;
 
     return 0;
diff --git a/libavutil/hwcontext_d3d11va.h b/libavutil/hwcontext_d3d11va.h
index 676349d7b8..d41451580e 100644
--- a/libavutil/hwcontext_d3d11va.h
+++ b/libavutil/hwcontext_d3d11va.h
@@ -26,6 +26,10 @@
  * The default pool implementation will be fixed-size if initial_pool_size is
  * set (and allocate elements from an array texture). Otherwise it will allocate
  * individual textures. Be aware that decoding requires a single array texture.
+ *
+ * Using sw_format==AV_PIX_FMT_YUV420P has special semantics, and maps to
+ * DXGI_FORMAT_420_OPAQUE. av_hwframe_transfer_data() is not supported for
+ * this format. Refer to MSDN for details.
  */
 
 #include <d3d11.h>