ffmpeg_cuvid: adapt for recent filter graph initialization changes

1 files changed, 30 insertions, 111 deletions

diff --git a/ffmpeg_cuvid.c b/ffmpeg_cuvid.c
index 46540795d4..3ff3b40f17 100644
--- a/ffmpeg_cuvid.c
+++ b/ffmpeg_cuvid.c
@@ -17,138 +17,57 @@
  */
 
 #include "libavutil/hwcontext.h"
+#include "libavutil/pixdesc.h"
 
 #include "ffmpeg.h"
 
-typedef struct CUVIDContext {
-    AVBufferRef *hw_frames_ctx;
-} CUVIDContext;
-
 static void cuvid_uninit(AVCodecContext *avctx)
 {
-    InputStream  *ist = avctx->opaque;
-    CUVIDContext *ctx = ist->hwaccel_ctx;
-
-    if (ctx) {
-        av_buffer_unref(&ctx->hw_frames_ctx);
-        av_freep(&ctx);
-    }
-
+    InputStream *ist = avctx->opaque;
     av_buffer_unref(&ist->hw_frames_ctx);
-
-    ist->hwaccel_ctx = 0;
-    ist->hwaccel_uninit = 0;
 }
 
 int cuvid_init(AVCodecContext *avctx)
 {
-    InputStream  *ist = avctx->opaque;
-    CUVIDContext *ctx = ist->hwaccel_ctx;
-
-    av_log(NULL, AV_LOG_TRACE, "Initializing cuvid hwaccel\n");
-
-    if (!ctx) {
-        av_log(NULL, AV_LOG_ERROR, "CUVID transcoding is not initialized. "
-               "-hwaccel cuvid should only be used for one-to-one CUVID transcoding "
-               "with no (software) filters.\n");
-        return AVERROR(EINVAL);
-    }
-
-    return 0;
-}
-
-int cuvid_transcode_init(OutputStream *ost)
-{
-    InputStream *ist;
-    const enum AVPixelFormat *pix_fmt;
-    AVHWFramesContext *hwframe_ctx;
-    AVBufferRef *device_ref = NULL;
-    CUVIDContext *ctx = NULL;
-    int ret = 0;
-
-    av_log(NULL, AV_LOG_TRACE, "Initializing cuvid transcoding\n");
+    InputStream *ist = avctx->opaque;
+    AVHWFramesContext *frames_ctx;
+    int ret;
 
-    if (ost->source_index < 0)
-        return 0;
+    av_log(avctx, AV_LOG_VERBOSE, "Initializing cuvid hwaccel\n");
 
-    ist = input_streams[ost->source_index];
-
-    /* check if the encoder supports CUVID */
-    if (!ost->enc->pix_fmts)
-        goto cancel;
-    for (pix_fmt = ost->enc->pix_fmts; *pix_fmt != AV_PIX_FMT_NONE; pix_fmt++)
-        if (*pix_fmt == AV_PIX_FMT_CUDA)
-            break;
-    if (*pix_fmt == AV_PIX_FMT_NONE)
-        goto cancel;
-
-    /* check if the decoder supports CUVID */
-    if (ist->hwaccel_id != HWACCEL_CUVID || !ist->dec || !ist->dec->pix_fmts)
-        goto cancel;
-    for (pix_fmt = ist->dec->pix_fmts; *pix_fmt != AV_PIX_FMT_NONE; pix_fmt++)
-        if (*pix_fmt == AV_PIX_FMT_CUDA)
-            break;
-    if (*pix_fmt == AV_PIX_FMT_NONE)
-        goto cancel;
-
-    av_log(NULL, AV_LOG_VERBOSE, "Setting up CUVID transcoding\n");
-
-    if (ist->hwaccel_ctx) {
-        ctx = ist->hwaccel_ctx;
-    } else {
-        ctx = av_mallocz(sizeof(*ctx));
-        if (!ctx) {
-            ret = AVERROR(ENOMEM);
-            goto error;
+    if (!hw_device_ctx) {
+        ret = av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_CUDA,
+                                     ist->hwaccel_device, NULL, 0);
+        if (ret < 0) {
+            av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA device\n");
+            return ret;
         }
     }
 
-    if (!ctx->hw_frames_ctx) {
-        ret = av_hwdevice_ctx_create(&device_ref, AV_HWDEVICE_TYPE_CUDA,
-                                     ist->hwaccel_device, NULL, 0);
-        if (ret < 0)
-            goto error;
+    av_buffer_unref(&ist->hw_frames_ctx);
+    ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
+    if (!ist->hw_frames_ctx) {
+        av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n");
+        return AVERROR(ENOMEM);
+    }
 
-        ctx->hw_frames_ctx = av_hwframe_ctx_alloc(device_ref);
-        if (!ctx->hw_frames_ctx) {
-            av_log(NULL, AV_LOG_ERROR, "av_hwframe_ctx_alloc failed\n");
-            ret = AVERROR(ENOMEM);
-            goto error;
-        }
-        av_buffer_unref(&device_ref);
+    frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data;
 
-        ist->hw_frames_ctx = av_buffer_ref(ctx->hw_frames_ctx);
-        if (!ist->hw_frames_ctx) {
-            av_log(NULL, AV_LOG_ERROR, "av_buffer_ref failed\n");
-            ret = AVERROR(ENOMEM);
-            goto error;
-        }
+    frames_ctx->format = AV_PIX_FMT_CUDA;
+    frames_ctx->sw_format = avctx->sw_pix_fmt;
+    frames_ctx->width = avctx->width;
+    frames_ctx->height = avctx->height;
 
-        ist->hwaccel_ctx = ctx;
-        ist->hwaccel_uninit = cuvid_uninit;
+    av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n",
+           av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height);
 
-        /* This is a bit hacky, av_hwframe_ctx_init is called by the cuvid decoder
-         * once it has probed the necessary format information. But as filters/nvenc
-         * need to know the format/sw_format, set them here so they are happy.
-         * This is fine as long as CUVID doesn't add another supported pix_fmt.
-         */
-        hwframe_ctx = (AVHWFramesContext*)ctx->hw_frames_ctx->data;
-        hwframe_ctx->format = AV_PIX_FMT_CUDA;
-        hwframe_ctx->sw_format = ist->st->codecpar->format == AV_PIX_FMT_YUV420P10 ? AV_PIX_FMT_P010 : AV_PIX_FMT_NV12;
+    ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
+    if (ret < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n");
+        return ret;
     }
 
-    return 0;
-
-error:
-    av_freep(&ctx);
-    av_buffer_unref(&device_ref);
-    return ret;
-
-cancel:
-    if (ist->hwaccel_id == HWACCEL_CUVID) {
-        av_log(NULL, AV_LOG_ERROR, "CUVID hwaccel requested, but impossible to achieve.\n");
-        return AVERROR(EINVAL);
-    }
+    ist->hwaccel_uninit = cuvid_uninit;
 
     return 0;
 }