lavc/qsvdec: require a dynamic frame pool if possible

This allows a downstream element stores more frames from qsv decoders
and fixes error in get_buffer().

$ ffmpeg -hwaccel qsv -hwaccel_output_format qsv -i input.mp4 -vf
reverse -f null -

[vist#0:0/h264 @ 0x562248f12c50] Decoding error: Cannot allocate memory
[h264_qsv @ 0x562248f66b10] get_buffer() failed

Signed-off-by: Haihao Xiang <haihao.xiang@intel.com>

1 files changed, 45 insertions, 12 deletions

diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
index a51ddace62..df0d49bc10 100644
--- a/libavcodec/qsvdec.c
+++ b/libavcodec/qsvdec.c
@@ -42,6 +42,7 @@
 #include "libavutil/imgutils.h"
 #include "libavutil/film_grain_params.h"
 #include "libavutil/mastering_display_metadata.h"
+#include "libavutil/avassert.h"
 
 #include "avcodec.h"
 #include "codec_internal.h"
@@ -68,6 +69,8 @@ static const AVRational mfx_tb = { 1, 90000 };
     AV_NOPTS_VALUE : pts_tb.num ? \
     av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
 
+#define MFX_IMPL_VIA_MASK(impl) (0x0f00 & (impl))
+
 typedef struct QSVAsyncFrame {
     mfxSyncPoint *sync;
     QSVFrame     *frame;
@@ -77,6 +80,7 @@ typedef struct QSVContext {
     // the session used for decoding
     mfxSession session;
     mfxVersion ver;
+    mfxHandleType handle_type;
 
     // the session we allocated internally, in case the caller did not provide
     // one
@@ -183,6 +187,7 @@ static int qsv_init_session(AVCodecContext *avctx, QSVContext *q, mfxSession ses
                             AVBufferRef *hw_frames_ref, AVBufferRef *hw_device_ref)
 {
     int ret;
+    mfxIMPL impl;
 
     if (q->gpu_copy == MFX_GPUCOPY_ON &&
         !(q->iopattern & MFX_IOPATTERN_OUT_SYSTEM_MEMORY)) {
@@ -240,27 +245,52 @@ static int qsv_init_session(AVCodecContext *avctx, QSVContext *q, mfxSession ses
         q->session = q->internal_qs.session;
     }
 
-    if (MFXQueryVersion(q->session, &q->ver) != MFX_ERR_NONE) {
-        av_log(avctx, AV_LOG_ERROR, "Error querying the session version. \n");
-        q->session = NULL;
+    if (MFXQueryIMPL(q->session, &impl) == MFX_ERR_NONE) {
+        switch (MFX_IMPL_VIA_MASK(impl)) {
+        case MFX_IMPL_VIA_VAAPI:
+            q->handle_type = MFX_HANDLE_VA_DISPLAY;
+            break;
 
-        if (q->internal_qs.session) {
-            MFXClose(q->internal_qs.session);
-            q->internal_qs.session = NULL;
-        }
+        case MFX_IMPL_VIA_D3D11:
+            q->handle_type = MFX_HANDLE_D3D11_DEVICE;
+            break;
+
+        case MFX_IMPL_VIA_D3D9:
+            q->handle_type = MFX_HANDLE_D3D9_DEVICE_MANAGER;
+            break;
 
-        if (q->internal_qs.loader) {
-            MFXUnload(q->internal_qs.loader);
-            q->internal_qs.loader = NULL;
+        default:
+            av_assert0(!"should not reach here");
         }
+    } else {
+        av_log(avctx, AV_LOG_ERROR, "Error querying the implementation. \n");
+        goto fail;
+    }
 
-        return AVERROR_EXTERNAL;
+    if (MFXQueryVersion(q->session, &q->ver) != MFX_ERR_NONE) {
+        av_log(avctx, AV_LOG_ERROR, "Error querying the session version. \n");
+        goto fail;
     }
 
     /* make sure the decoder is uninitialized */
     MFXVideoDECODE_Close(q->session);
 
     return 0;
+
+fail:
+    q->session = NULL;
+
+    if (q->internal_qs.session) {
+        MFXClose(q->internal_qs.session);
+        q->internal_qs.session = NULL;
+    }
+
+    if (q->internal_qs.loader) {
+        MFXUnload(q->internal_qs.loader);
+        q->internal_qs.loader = NULL;
+    }
+
+    return AVERROR_EXTERNAL;
 }
 
 static int qsv_decode_preinit(AVCodecContext *avctx, QSVContext *q, enum AVPixelFormat pix_fmt, mfxVideoParam *param)
@@ -310,7 +340,10 @@ static int qsv_decode_preinit(AVCodecContext *avctx, QSVContext *q, enum AVPixel
         hwframes_ctx->height            = FFALIGN(avctx->coded_height, 32);
         hwframes_ctx->format            = AV_PIX_FMT_QSV;
         hwframes_ctx->sw_format         = avctx->sw_pix_fmt;
-        hwframes_ctx->initial_pool_size = q->suggest_pool_size + 16 + avctx->extra_hw_frames;
+        if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 9) && q->handle_type != MFX_HANDLE_D3D9_DEVICE_MANAGER)
+            hwframes_ctx->initial_pool_size = 0;
+        else
+            hwframes_ctx->initial_pool_size = q->suggest_pool_size + 16 + avctx->extra_hw_frames;
         frames_hwctx->frame_type        = MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
 
         ret = av_hwframe_ctx_init(avctx->hw_frames_ctx);