fftools/ffmpeg_hw: move hw_device_setup_for_decode() to ffmpeg_dec

This function is entangled with decoder setup, so it is more decoding
code rather than ffmpeg_hw code. This will allow making more decoder
state private in the future.

3 files changed, 153 insertions, 150 deletions

diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 45be3b1823..895edbd6d6 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -795,10 +795,13 @@ void enc_stats_write(OutputStream *ost, EncStats *es,
                      uint64_t frame_num);
 
 HWDevice *hw_device_get_by_name(const char *name);
+HWDevice *hw_device_get_by_type(enum AVHWDeviceType type);
 int hw_device_init_from_string(const char *arg, HWDevice **dev);
+int hw_device_init_from_type(enum AVHWDeviceType type,
+                             const char *device,
+                             HWDevice **dev_out);
 void hw_device_free_all(void);
 
-int hw_device_setup_for_decode(InputStream *ist);
 int hw_device_setup_for_encode(OutputStream *ost);
 /**
  * Get a hardware device to be used with this filtergraph.
diff --git a/fftools/ffmpeg_dec.c b/fftools/ffmpeg_dec.c
index 04b7db1e17..8946bf538d 100644
--- a/fftools/ffmpeg_dec.c
+++ b/fftools/ffmpeg_dec.c
@@ -533,6 +533,151 @@ static enum AVPixelFormat get_format(AVCodecContext *s, const enum AVPixelFormat
     return *p;
 }
 
+static HWDevice *hw_device_match_by_codec(const AVCodec *codec)
+{
+    const AVCodecHWConfig *config;
+    HWDevice *dev;
+    int i;
+    for (i = 0;; i++) {
+        config = avcodec_get_hw_config(codec, i);
+        if (!config)
+            return NULL;
+        if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX))
+            continue;
+        dev = hw_device_get_by_type(config->device_type);
+        if (dev)
+            return dev;
+    }
+}
+
+static int hw_device_setup_for_decode(InputStream *ist)
+{
+    const AVCodecHWConfig *config;
+    enum AVHWDeviceType type;
+    HWDevice *dev = NULL;
+    int err, auto_device = 0;
+
+    if (ist->hwaccel_device) {
+        dev = hw_device_get_by_name(ist->hwaccel_device);
+        if (!dev) {
+            if (ist->hwaccel_id == HWACCEL_AUTO) {
+                auto_device = 1;
+            } else if (ist->hwaccel_id == HWACCEL_GENERIC) {
+                type = ist->hwaccel_device_type;
+                err = hw_device_init_from_type(type, ist->hwaccel_device,
+                                               &dev);
+            } else {
+                // This will be dealt with by API-specific initialisation
+                // (using hwaccel_device), so nothing further needed here.
+                return 0;
+            }
+        } else {
+            if (ist->hwaccel_id == HWACCEL_AUTO) {
+                ist->hwaccel_device_type = dev->type;
+            } else if (ist->hwaccel_device_type != dev->type) {
+                av_log(NULL, AV_LOG_ERROR, "Invalid hwaccel device "
+                       "specified for decoder: device %s of type %s is not "
+                       "usable with hwaccel %s.\n", dev->name,
+                       av_hwdevice_get_type_name(dev->type),
+                       av_hwdevice_get_type_name(ist->hwaccel_device_type));
+                return AVERROR(EINVAL);
+            }
+        }
+    } else {
+        if (ist->hwaccel_id == HWACCEL_AUTO) {
+            auto_device = 1;
+        } else if (ist->hwaccel_id == HWACCEL_GENERIC) {
+            type = ist->hwaccel_device_type;
+            dev = hw_device_get_by_type(type);
+
+            // When "-qsv_device device" is used, an internal QSV device named
+            // as "__qsv_device" is created. Another QSV device is created too
+            // if "-init_hw_device qsv=name:device" is used. There are 2 QSV devices
+            // if both "-qsv_device device" and "-init_hw_device qsv=name:device"
+            // are used, hw_device_get_by_type(AV_HWDEVICE_TYPE_QSV) returns NULL.
+            // To keep back-compatibility with the removed ad-hoc libmfx setup code,
+            // call hw_device_get_by_name("__qsv_device") to select the internal QSV
+            // device.
+            if (!dev && type == AV_HWDEVICE_TYPE_QSV)
+                dev = hw_device_get_by_name("__qsv_device");
+
+            if (!dev)
+                err = hw_device_init_from_type(type, NULL, &dev);
+        } else {
+            dev = hw_device_match_by_codec(ist->dec);
+            if (!dev) {
+                // No device for this codec, but not using generic hwaccel
+                // and therefore may well not need one - ignore.
+                return 0;
+            }
+        }
+    }
+
+    if (auto_device) {
+        int i;
+        if (!avcodec_get_hw_config(ist->dec, 0)) {
+            // Decoder does not support any hardware devices.
+            return 0;
+        }
+        for (i = 0; !dev; i++) {
+            config = avcodec_get_hw_config(ist->dec, i);
+            if (!config)
+                break;
+            type = config->device_type;
+            dev = hw_device_get_by_type(type);
+            if (dev) {
+                av_log(NULL, AV_LOG_INFO, "Using auto "
+                       "hwaccel type %s with existing device %s.\n",
+                       av_hwdevice_get_type_name(type), dev->name);
+            }
+        }
+        for (i = 0; !dev; i++) {
+            config = avcodec_get_hw_config(ist->dec, i);
+            if (!config)
+                break;
+            type = config->device_type;
+            // Try to make a new device of this type.
+            err = hw_device_init_from_type(type, ist->hwaccel_device,
+                                           &dev);
+            if (err < 0) {
+                // Can't make a device of this type.
+                continue;
+            }
+            if (ist->hwaccel_device) {
+                av_log(NULL, AV_LOG_INFO, "Using auto "
+                       "hwaccel type %s with new device created "
+                       "from %s.\n", av_hwdevice_get_type_name(type),
+                       ist->hwaccel_device);
+            } else {
+                av_log(NULL, AV_LOG_INFO, "Using auto "
+                       "hwaccel type %s with new default device.\n",
+                       av_hwdevice_get_type_name(type));
+            }
+        }
+        if (dev) {
+            ist->hwaccel_device_type = type;
+        } else {
+            av_log(NULL, AV_LOG_INFO, "Auto hwaccel "
+                   "disabled: no device found.\n");
+            ist->hwaccel_id = HWACCEL_NONE;
+            return 0;
+        }
+    }
+
+    if (!dev) {
+        av_log(NULL, AV_LOG_ERROR, "No device available "
+               "for decoder: device type %s needed for codec %s.\n",
+               av_hwdevice_get_type_name(type), ist->dec->name);
+        return err;
+    }
+
+    ist->dec_ctx->hw_device_ctx = av_buffer_ref(dev->device_ref);
+    if (!ist->dec_ctx->hw_device_ctx)
+        return AVERROR(ENOMEM);
+
+    return 0;
+}
+
 int dec_open(InputStream *ist)
 {
     const AVCodec *codec = ist->dec;
diff --git a/fftools/ffmpeg_hw.c b/fftools/ffmpeg_hw.c
index e67145211c..d28257a1d6 100644
--- a/fftools/ffmpeg_hw.c
+++ b/fftools/ffmpeg_hw.c
@@ -27,7 +27,7 @@
 static int nb_hw_devices;
 static HWDevice **hw_devices;
 
-static HWDevice *hw_device_get_by_type(enum AVHWDeviceType type)
+HWDevice *hw_device_get_by_type(enum AVHWDeviceType type)
 {
     HWDevice *found = NULL;
     int i;
@@ -242,9 +242,9 @@ fail:
     goto done;
 }
 
-static int hw_device_init_from_type(enum AVHWDeviceType type,
-                                    const char *device,
-                                    HWDevice **dev_out)
+int hw_device_init_from_type(enum AVHWDeviceType type,
+                             const char *device,
+                             HWDevice **dev_out)
 {
     AVBufferRef *device_ref = NULL;
     HWDevice *dev;
@@ -297,151 +297,6 @@ void hw_device_free_all(void)
     nb_hw_devices = 0;
 }
 
-static HWDevice *hw_device_match_by_codec(const AVCodec *codec)
-{
-    const AVCodecHWConfig *config;
-    HWDevice *dev;
-    int i;
-    for (i = 0;; i++) {
-        config = avcodec_get_hw_config(codec, i);
-        if (!config)
-            return NULL;
-        if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX))
-            continue;
-        dev = hw_device_get_by_type(config->device_type);
-        if (dev)
-            return dev;
-    }
-}
-
-int hw_device_setup_for_decode(InputStream *ist)
-{
-    const AVCodecHWConfig *config;
-    enum AVHWDeviceType type;
-    HWDevice *dev = NULL;
-    int err, auto_device = 0;
-
-    if (ist->hwaccel_device) {
-        dev = hw_device_get_by_name(ist->hwaccel_device);
-        if (!dev) {
-            if (ist->hwaccel_id == HWACCEL_AUTO) {
-                auto_device = 1;
-            } else if (ist->hwaccel_id == HWACCEL_GENERIC) {
-                type = ist->hwaccel_device_type;
-                err = hw_device_init_from_type(type, ist->hwaccel_device,
-                                               &dev);
-            } else {
-                // This will be dealt with by API-specific initialisation
-                // (using hwaccel_device), so nothing further needed here.
-                return 0;
-            }
-        } else {
-            if (ist->hwaccel_id == HWACCEL_AUTO) {
-                ist->hwaccel_device_type = dev->type;
-            } else if (ist->hwaccel_device_type != dev->type) {
-                av_log(NULL, AV_LOG_ERROR, "Invalid hwaccel device "
-                       "specified for decoder: device %s of type %s is not "
-                       "usable with hwaccel %s.\n", dev->name,
-                       av_hwdevice_get_type_name(dev->type),
-                       av_hwdevice_get_type_name(ist->hwaccel_device_type));
-                return AVERROR(EINVAL);
-            }
-        }
-    } else {
-        if (ist->hwaccel_id == HWACCEL_AUTO) {
-            auto_device = 1;
-        } else if (ist->hwaccel_id == HWACCEL_GENERIC) {
-            type = ist->hwaccel_device_type;
-            dev = hw_device_get_by_type(type);
-
-            // When "-qsv_device device" is used, an internal QSV device named
-            // as "__qsv_device" is created. Another QSV device is created too
-            // if "-init_hw_device qsv=name:device" is used. There are 2 QSV devices
-            // if both "-qsv_device device" and "-init_hw_device qsv=name:device"
-            // are used, hw_device_get_by_type(AV_HWDEVICE_TYPE_QSV) returns NULL.
-            // To keep back-compatibility with the removed ad-hoc libmfx setup code,
-            // call hw_device_get_by_name("__qsv_device") to select the internal QSV
-            // device.
-            if (!dev && type == AV_HWDEVICE_TYPE_QSV)
-                dev = hw_device_get_by_name("__qsv_device");
-
-            if (!dev)
-                err = hw_device_init_from_type(type, NULL, &dev);
-        } else {
-            dev = hw_device_match_by_codec(ist->dec);
-            if (!dev) {
-                // No device for this codec, but not using generic hwaccel
-                // and therefore may well not need one - ignore.
-                return 0;
-            }
-        }
-    }
-
-    if (auto_device) {
-        int i;
-        if (!avcodec_get_hw_config(ist->dec, 0)) {
-            // Decoder does not support any hardware devices.
-            return 0;
-        }
-        for (i = 0; !dev; i++) {
-            config = avcodec_get_hw_config(ist->dec, i);
-            if (!config)
-                break;
-            type = config->device_type;
-            dev = hw_device_get_by_type(type);
-            if (dev) {
-                av_log(NULL, AV_LOG_INFO, "Using auto "
-                       "hwaccel type %s with existing device %s.\n",
-                       av_hwdevice_get_type_name(type), dev->name);
-            }
-        }
-        for (i = 0; !dev; i++) {
-            config = avcodec_get_hw_config(ist->dec, i);
-            if (!config)
-                break;
-            type = config->device_type;
-            // Try to make a new device of this type.
-            err = hw_device_init_from_type(type, ist->hwaccel_device,
-                                           &dev);
-            if (err < 0) {
-                // Can't make a device of this type.
-                continue;
-            }
-            if (ist->hwaccel_device) {
-                av_log(NULL, AV_LOG_INFO, "Using auto "
-                       "hwaccel type %s with new device created "
-                       "from %s.\n", av_hwdevice_get_type_name(type),
-                       ist->hwaccel_device);
-            } else {
-                av_log(NULL, AV_LOG_INFO, "Using auto "
-                       "hwaccel type %s with new default device.\n",
-                       av_hwdevice_get_type_name(type));
-            }
-        }
-        if (dev) {
-            ist->hwaccel_device_type = type;
-        } else {
-            av_log(NULL, AV_LOG_INFO, "Auto hwaccel "
-                   "disabled: no device found.\n");
-            ist->hwaccel_id = HWACCEL_NONE;
-            return 0;
-        }
-    }
-
-    if (!dev) {
-        av_log(NULL, AV_LOG_ERROR, "No device available "
-               "for decoder: device type %s needed for codec %s.\n",
-               av_hwdevice_get_type_name(type), ist->dec->name);
-        return err;
-    }
-
-    ist->dec_ctx->hw_device_ctx = av_buffer_ref(dev->device_ref);
-    if (!ist->dec_ctx->hw_device_ctx)
-        return AVERROR(ENOMEM);
-
-    return 0;
-}
-
 int hw_device_setup_for_encode(OutputStream *ost)
 {
     const AVCodecHWConfig *config;