ffmpeg_cuvid: adapt for recent filter graph initialization changes

3 files changed, 30 insertions, 125 deletions

diff --git a/ffmpeg.c b/ffmpeg.c
index 275894dbf5..6e57524558 100644
--- a/ffmpeg.c
+++ b/ffmpeg.c
@@ -3546,19 +3546,6 @@ static int transcode_init(void)
                 input_streams[j + ifile->ist_index]->start = av_gettime_relative();
     }
 
-    /* hwaccel transcoding */
-    for (i = 0; i < nb_output_streams; i++) {
-        ost = output_streams[i];
-
-        if (!ost->stream_copy) {
-
-#if CONFIG_CUVID
-            if (cuvid_transcode_init(ost))
-                exit_program(1);
-#endif
-        }
-    }
-
     /* init input streams */
     for (i = 0; i < nb_input_streams; i++)
         if ((ret = init_input_stream(i, error, sizeof(error))) < 0) {
diff --git a/ffmpeg.h b/ffmpeg.h
index 5d20d453eb..59f6cb3659 100644
--- a/ffmpeg.h
+++ b/ffmpeg.h
@@ -648,6 +648,5 @@ int qsv_init(AVCodecContext *s);
 int vaapi_decode_init(AVCodecContext *avctx);
 int vaapi_device_init(const char *device);
 int cuvid_init(AVCodecContext *s);
-int cuvid_transcode_init(OutputStream *ost);
 
 #endif /* FFMPEG_H */
diff --git a/ffmpeg_cuvid.c b/ffmpeg_cuvid.c
index 46540795d4..3ff3b40f17 100644
--- a/ffmpeg_cuvid.c
+++ b/ffmpeg_cuvid.c
@@ -17,138 +17,57 @@
  */
 
 #include "libavutil/hwcontext.h"
+#include "libavutil/pixdesc.h"
 
 #include "ffmpeg.h"
 
-typedef struct CUVIDContext {
-    AVBufferRef *hw_frames_ctx;
-} CUVIDContext;
-
 static void cuvid_uninit(AVCodecContext *avctx)
 {
-    InputStream  *ist = avctx->opaque;
-    CUVIDContext *ctx = ist->hwaccel_ctx;
-
-    if (ctx) {
-        av_buffer_unref(&ctx->hw_frames_ctx);
-        av_freep(&ctx);
-    }
-
+    InputStream *ist = avctx->opaque;
     av_buffer_unref(&ist->hw_frames_ctx);
-
-    ist->hwaccel_ctx = 0;
-    ist->hwaccel_uninit = 0;
 }
 
 int cuvid_init(AVCodecContext *avctx)
 {
-    InputStream  *ist = avctx->opaque;
-    CUVIDContext *ctx = ist->hwaccel_ctx;
-
-    av_log(NULL, AV_LOG_TRACE, "Initializing cuvid hwaccel\n");
-
-    if (!ctx) {
-        av_log(NULL, AV_LOG_ERROR, "CUVID transcoding is not initialized. "
-               "-hwaccel cuvid should only be used for one-to-one CUVID transcoding "
-               "with no (software) filters.\n");
-        return AVERROR(EINVAL);
-    }
-
-    return 0;
-}
-
-int cuvid_transcode_init(OutputStream *ost)
-{
-    InputStream *ist;
-    const enum AVPixelFormat *pix_fmt;
-    AVHWFramesContext *hwframe_ctx;
-    AVBufferRef *device_ref = NULL;
-    CUVIDContext *ctx = NULL;
-    int ret = 0;
-
-    av_log(NULL, AV_LOG_TRACE, "Initializing cuvid transcoding\n");
+    InputStream *ist = avctx->opaque;
+    AVHWFramesContext *frames_ctx;
+    int ret;
 
-    if (ost->source_index < 0)
-        return 0;
+    av_log(avctx, AV_LOG_VERBOSE, "Initializing cuvid hwaccel\n");
 
-    ist = input_streams[ost->source_index];
-
-    /* check if the encoder supports CUVID */
-    if (!ost->enc->pix_fmts)
-        goto cancel;
-    for (pix_fmt = ost->enc->pix_fmts; *pix_fmt != AV_PIX_FMT_NONE; pix_fmt++)
-        if (*pix_fmt == AV_PIX_FMT_CUDA)
-            break;
-    if (*pix_fmt == AV_PIX_FMT_NONE)
-        goto cancel;
-
-    /* check if the decoder supports CUVID */
-    if (ist->hwaccel_id != HWACCEL_CUVID || !ist->dec || !ist->dec->pix_fmts)
-        goto cancel;
-    for (pix_fmt = ist->dec->pix_fmts; *pix_fmt != AV_PIX_FMT_NONE; pix_fmt++)
-        if (*pix_fmt == AV_PIX_FMT_CUDA)
-            break;
-    if (*pix_fmt == AV_PIX_FMT_NONE)
-        goto cancel;
-
-    av_log(NULL, AV_LOG_VERBOSE, "Setting up CUVID transcoding\n");
-
-    if (ist->hwaccel_ctx) {
-        ctx = ist->hwaccel_ctx;
-    } else {
-        ctx = av_mallocz(sizeof(*ctx));
-        if (!ctx) {
-            ret = AVERROR(ENOMEM);
-            goto error;
+    if (!hw_device_ctx) {
+        ret = av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_CUDA,
+                                     ist->hwaccel_device, NULL, 0);
+        if (ret < 0) {
+            av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA device\n");
+            return ret;
         }
     }
 
-    if (!ctx->hw_frames_ctx) {
-        ret = av_hwdevice_ctx_create(&device_ref, AV_HWDEVICE_TYPE_CUDA,
-                                     ist->hwaccel_device, NULL, 0);
-        if (ret < 0)
-            goto error;
+    av_buffer_unref(&ist->hw_frames_ctx);
+    ist->hw_frames_ctx = av_hwframe_ctx_alloc(hw_device_ctx);
+    if (!ist->hw_frames_ctx) {
+        av_log(avctx, AV_LOG_ERROR, "Error creating a CUDA frames context\n");
+        return AVERROR(ENOMEM);
+    }
 
-        ctx->hw_frames_ctx = av_hwframe_ctx_alloc(device_ref);
-        if (!ctx->hw_frames_ctx) {
-            av_log(NULL, AV_LOG_ERROR, "av_hwframe_ctx_alloc failed\n");
-            ret = AVERROR(ENOMEM);
-            goto error;
-        }
-        av_buffer_unref(&device_ref);
+    frames_ctx = (AVHWFramesContext*)ist->hw_frames_ctx->data;
 
-        ist->hw_frames_ctx = av_buffer_ref(ctx->hw_frames_ctx);
-        if (!ist->hw_frames_ctx) {
-            av_log(NULL, AV_LOG_ERROR, "av_buffer_ref failed\n");
-            ret = AVERROR(ENOMEM);
-            goto error;
-        }
+    frames_ctx->format = AV_PIX_FMT_CUDA;
+    frames_ctx->sw_format = avctx->sw_pix_fmt;
+    frames_ctx->width = avctx->width;
+    frames_ctx->height = avctx->height;
 
-        ist->hwaccel_ctx = ctx;
-        ist->hwaccel_uninit = cuvid_uninit;
+    av_log(avctx, AV_LOG_DEBUG, "Initializing CUDA frames context: sw_format = %s, width = %d, height = %d\n",
+           av_get_pix_fmt_name(frames_ctx->sw_format), frames_ctx->width, frames_ctx->height);
 
-        /* This is a bit hacky, av_hwframe_ctx_init is called by the cuvid decoder
-         * once it has probed the necessary format information. But as filters/nvenc
-         * need to know the format/sw_format, set them here so they are happy.
-         * This is fine as long as CUVID doesn't add another supported pix_fmt.
-         */
-        hwframe_ctx = (AVHWFramesContext*)ctx->hw_frames_ctx->data;
-        hwframe_ctx->format = AV_PIX_FMT_CUDA;
-        hwframe_ctx->sw_format = ist->st->codecpar->format == AV_PIX_FMT_YUV420P10 ? AV_PIX_FMT_P010 : AV_PIX_FMT_NV12;
+    ret = av_hwframe_ctx_init(ist->hw_frames_ctx);
+    if (ret < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Error initializing a CUDA frame pool\n");
+        return ret;
     }
 
-    return 0;
-
-error:
-    av_freep(&ctx);
-    av_buffer_unref(&device_ref);
-    return ret;
-
-cancel:
-    if (ist->hwaccel_id == HWACCEL_CUVID) {
-        av_log(NULL, AV_LOG_ERROR, "CUVID hwaccel requested, but impossible to achieve.\n");
-        return AVERROR(EINVAL);
-    }
+    ist->hwaccel_uninit = cuvid_uninit;
 
     return 0;
 }