ffmpeg: init filtergraphs only after we have a frame on each input

This makes sure the actual stream parameters are used, which is
important mainly for hardware decoding+filtering cases, which would
previously require various weird workarounds to handle the fact that a
fake software graph has to be constructed, but never used.
This should also improve behaviour in rare cases where
avformat_find_stream_info() does not provide accurate information.

This merges Libav commit a3a0230. It was previously skipped.

The code in flush_encoders() which sets up a "fake" format wasn't in
Libav. I'm not sure if it's a good idea, but it tends to give
behavior closer to the old one in certain corner cases.

The vp8-size-change gives different result, because now the size of
the first frame is used. libavformat reported the size of the largest
frame for some reason.

The exr tests now use the sample aspect ratio of the first frame. For
some reason libavformat determines 0/1 as aspect ratio, while the
decoder returns the correct one.

The ffm and mxf tests change the field_order values. I'm assuming
another libavformat/decoding mismatch.

Signed-off-by: wm4 <nfxjfg@googlemail.com>

18 files changed, 281 insertions, 337 deletions

diff --git a/ffmpeg.c b/ffmpeg.c
index 983e2fb486..88f68342ed 100644
--- a/ffmpeg.c
+++ b/ffmpeg.c
@@ -123,6 +123,7 @@ const char *const forced_keyframes_const_names[] = {
 static void do_video_stats(OutputStream *ost, int frame_size);
 static int64_t getutime(void);
 static int64_t getmaxrss(void);
+static int ifilter_has_all_input_formats(FilterGraph *fg);
 
 static int run_as_daemon  = 0;
 static int nb_frames_dup = 0;
@@ -472,6 +473,13 @@ static void ffmpeg_cleanup(int ret)
         FilterGraph *fg = filtergraphs[i];
         avfilter_graph_free(&fg->graph);
         for (j = 0; j < fg->nb_inputs; j++) {
+            while (av_fifo_size(fg->inputs[j]->frame_queue)) {
+                AVFrame *frame;
+                av_fifo_generic_read(fg->inputs[j]->frame_queue, &frame,
+                                     sizeof(frame), NULL);
+                av_frame_free(&frame);
+            }
+            av_fifo_free(fg->inputs[j]->frame_queue);
             av_buffer_unref(&fg->inputs[j]->hw_frames_ctx);
             av_freep(&fg->inputs[j]->name);
             av_freep(&fg->inputs[j]);
@@ -1377,6 +1385,8 @@ static void do_video_stats(OutputStream *ost, int frame_size)
     }
 }
 
+static int init_output_stream(OutputStream *ost, char *error, int error_len);
+
 static void finish_output_stream(OutputStream *ost)
 {
     OutputFile *of = output_files[ost->file_index];
@@ -1409,10 +1419,20 @@ static int reap_filters(int flush)
         AVCodecContext *enc = ost->enc_ctx;
         int ret = 0;
 
-        if (!ost->filter)
+        if (!ost->filter || !ost->filter->graph->graph)
             continue;
         filter = ost->filter->filter;
 
+        if (!ost->initialized) {
+            char error[1024];
+            ret = init_output_stream(ost, error, sizeof(error));
+            if (ret < 0) {
+                av_log(NULL, AV_LOG_ERROR, "Error initializing output stream %d:%d -- %s\n",
+                       ost->file_index, ost->index, error);
+                exit_program(1);
+            }
+        }
+
         if (!ost->filtered_frame && !(ost->filtered_frame = av_frame_alloc())) {
             return AVERROR(ENOMEM);
         }
@@ -1813,6 +1833,54 @@ static void flush_encoders(void)
         if (!ost->encoding_needed)
             continue;
 
+        // Try to enable encoding with no input frames.
+        // Maybe we should just let encoding fail instead.
+        if (!ost->initialized) {
+            FilterGraph *fg = ost->filter->graph;
+            char error[1024];
+
+            av_log(NULL, AV_LOG_WARNING,
+                   "Finishing stream %d:%d without any data written to it.\n",
+                   ost->file_index, ost->st->index);
+
+            if (ost->filter && !fg->graph) {
+                int x;
+                for (x = 0; x < fg->nb_inputs; x++) {
+                    InputFilter *ifilter = fg->inputs[x];
+                    if (ifilter->format < 0) {
+                        AVCodecParameters *par = ifilter->ist->st->codecpar;
+                        // We never got any input. Set a fake format, which will
+                        // come from libavformat.
+                        ifilter->format                 = par->format;
+                        ifilter->sample_rate            = par->sample_rate;
+                        ifilter->channels               = par->channels;
+                        ifilter->channel_layout         = par->channel_layout;
+                        ifilter->width                  = par->width;
+                        ifilter->height                 = par->height;
+                        ifilter->sample_aspect_ratio    = par->sample_aspect_ratio;
+                    }
+                }
+
+                if (!ifilter_has_all_input_formats(fg))
+                    continue;
+
+                ret = configure_filtergraph(fg);
+                if (ret < 0) {
+                    av_log(NULL, AV_LOG_ERROR, "Error configuring filter graph\n");
+                    exit_program(1);
+                }
+
+                finish_output_stream(ost);
+            }
+
+            ret = init_output_stream(ost, error, sizeof(error));
+            if (ret < 0) {
+                av_log(NULL, AV_LOG_ERROR, "Error initializing output stream %d:%d -- %s\n",
+                       ost->file_index, ost->index, error);
+                exit_program(1);
+            }
+        }
+
         if (enc->codec_type == AVMEDIA_TYPE_AUDIO && enc->frame_size <= 1)
             continue;
 #if FF_API_LAVF_FMT_RAWPICTURE
@@ -2044,6 +2112,102 @@ static void check_decode_result(InputStream *ist, int *got_output, int ret)
     }
 }
 
+// Filters can be configured only if the formats of all inputs are known.
+static int ifilter_has_all_input_formats(FilterGraph *fg)
+{
+    int i;
+    for (i = 0; i < fg->nb_inputs; i++) {
+        if (fg->inputs[i]->format < 0 && (fg->inputs[i]->type == AVMEDIA_TYPE_AUDIO ||
+                                          fg->inputs[i]->type == AVMEDIA_TYPE_VIDEO))
+            return 0;
+    }
+    return 1;
+}
+
+static int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame)
+{
+    FilterGraph *fg = ifilter->graph;
+    int need_reinit, ret, i;
+
+    /* determine if the parameters for this input changed */
+    need_reinit = ifilter->format != frame->format;
+    if (!!ifilter->hw_frames_ctx != !!frame->hw_frames_ctx ||
+        (ifilter->hw_frames_ctx && ifilter->hw_frames_ctx->data != frame->hw_frames_ctx->data))
+        need_reinit = 1;
+
+    switch (ifilter->ist->st->codecpar->codec_type) {
+    case AVMEDIA_TYPE_AUDIO:
+        need_reinit |= ifilter->sample_rate    != frame->sample_rate ||
+                       ifilter->channels       != frame->channels ||
+                       ifilter->channel_layout != frame->channel_layout;
+        break;
+    case AVMEDIA_TYPE_VIDEO:
+        need_reinit |= ifilter->width  != frame->width ||
+                       ifilter->height != frame->height;
+        break;
+    }
+
+    if (need_reinit) {
+        ret = ifilter_parameters_from_frame(ifilter, frame);
+        if (ret < 0)
+            return ret;
+    }
+
+    /* (re)init the graph if possible, otherwise buffer the frame and return */
+    if (need_reinit || !fg->graph) {
+        for (i = 0; i < fg->nb_inputs; i++) {
+            if (!ifilter_has_all_input_formats(fg)) {
+                AVFrame *tmp = av_frame_clone(frame);
+                if (!tmp)
+                    return AVERROR(ENOMEM);
+                av_frame_unref(frame);
+
+                if (!av_fifo_space(ifilter->frame_queue)) {
+                    ret = av_fifo_realloc2(ifilter->frame_queue, 2 * av_fifo_size(ifilter->frame_queue));
+                    if (ret < 0)
+                        return ret;
+                }
+                av_fifo_generic_write(ifilter->frame_queue, &tmp, sizeof(tmp), NULL);
+                return 0;
+            }
+        }
+
+        ret = reap_filters(1);
+        if (ret < 0 && ret != AVERROR_EOF) {
+            char errbuf[128];
+            av_strerror(ret, errbuf, sizeof(errbuf));
+
+            av_log(NULL, AV_LOG_ERROR, "Error while filtering: %s\n", errbuf);
+            return ret;
+        }
+
+        ret = configure_filtergraph(fg);
+        if (ret < 0) {
+            av_log(NULL, AV_LOG_ERROR, "Error reinitializing filters!\n");
+            return ret;
+        }
+
+        for (i = 0; i < fg->nb_inputs; i++) {
+            while (av_fifo_size(fg->inputs[i]->frame_queue)) {
+                AVFrame *tmp;
+                av_fifo_generic_read(fg->inputs[i]->frame_queue, &tmp, sizeof(tmp), NULL);
+                ret = av_buffersrc_add_frame(fg->inputs[i]->filter, tmp);
+                av_frame_free(&tmp);
+                if (ret < 0)
+                    return ret;
+            }
+        }
+    }
+
+    ret = av_buffersrc_add_frame_flags(ifilter->filter, frame, AV_BUFFERSRC_FLAG_PUSH);
+    if (ret < 0) {
+        av_log(NULL, AV_LOG_ERROR, "Error while filtering\n");
+        return ret;
+    }
+
+    return 0;
+}
+
 // This does not quite work like avcodec_decode_audio4/avcodec_decode_video2.
 // There is the following difference: if you got a frame, you must call
 // it again with pkt=NULL. pkt==NULL is treated differently from pkt.size==0
@@ -2085,8 +2249,7 @@ static int send_frame_to_filters(InputStream *ist, AVFrame *decoded_frame)
                 break;
         } else
             f = decoded_frame;
-        ret = av_buffersrc_add_frame_flags(ist->filters[i]->filter, f,
-                                           AV_BUFFERSRC_FLAG_PUSH);
+        ret = ifilter_send_frame(ist->filters[i], f);
         if (ret == AVERROR_EOF)
             ret = 0; /* ignore */
         if (ret < 0) {
@@ -2102,7 +2265,7 @@ static int decode_audio(InputStream *ist, AVPacket *pkt, int *got_output)
 {
     AVFrame *decoded_frame;
     AVCodecContext *avctx = ist->dec_ctx;
-    int i, ret, err = 0, resample_changed;
+    int ret, err = 0;
     AVRational decoded_frame_tb;
 
     if (!ist->decoded_frame && !(ist->decoded_frame = av_frame_alloc()))
@@ -2138,59 +2301,6 @@ static int decode_audio(InputStream *ist, AVPacket *pkt, int *got_output)
                      avctx->sample_rate;
 #endif
 
-    resample_changed = ist->resample_sample_fmt     != decoded_frame->format         ||
-                       ist->resample_channels       != avctx->channels               ||
-                       ist->resample_channel_layout != decoded_frame->channel_layout ||
-                       ist->resample_sample_rate    != decoded_frame->sample_rate;
-    if (resample_changed) {
-        char layout1[64], layout2[64];
-
-        if (!guess_input_channel_layout(ist)) {
-            av_log(NULL, AV_LOG_FATAL, "Unable to find default channel "
-                   "layout for Input Stream #%d.%d\n", ist->file_index,
-                   ist->st->index);
-            exit_program(1);
-        }
-        decoded_frame->channel_layout = avctx->channel_layout;
-
-        av_get_channel_layout_string(layout1, sizeof(layout1), ist->resample_channels,
-                                     ist->resample_channel_layout);
-        av_get_channel_layout_string(layout2, sizeof(layout2), avctx->channels,
-                                     decoded_frame->channel_layout);
-
-        av_log(NULL, AV_LOG_INFO,
-               "Input stream #%d:%d frame changed from rate:%d fmt:%s ch:%d chl:%s to rate:%d fmt:%s ch:%d chl:%s\n",
-               ist->file_index, ist->st->index,
-               ist->resample_sample_rate,  av_get_sample_fmt_name(ist->resample_sample_fmt),
-               ist->resample_channels, layout1,
-               decoded_frame->sample_rate, av_get_sample_fmt_name(decoded_frame->format),
-               avctx->channels, layout2);
-
-        ist->resample_sample_fmt     = decoded_frame->format;
-        ist->resample_sample_rate    = decoded_frame->sample_rate;
-        ist->resample_channel_layout = decoded_frame->channel_layout;
-        ist->resample_channels       = avctx->channels;
-
-        for (i = 0; i < ist->nb_filters; i++) {
-            err = ifilter_parameters_from_frame(ist->filters[i], decoded_frame);
-            if (err < 0) {
-                av_log(NULL, AV_LOG_ERROR,
-                       "Error reconfiguring input stream %d:%d filter %d\n",
-                       ist->file_index, ist->st->index, i);
-                goto fail;
-            }
-        }
-
-        for (i = 0; i < nb_filtergraphs; i++)
-            if (ist_in_filtergraph(filtergraphs[i], ist)) {
-                FilterGraph *fg = filtergraphs[i];
-                if (configure_filtergraph(fg) < 0) {
-                    av_log(NULL, AV_LOG_FATAL, "Error reinitializing filters!\n");
-                    exit_program(1);
-                }
-            }
-    }
-
     if (decoded_frame->pts != AV_NOPTS_VALUE) {
         decoded_frame_tb   = ist->st->time_base;
     } else if (pkt && pkt->pts != AV_NOPTS_VALUE) {
@@ -2206,9 +2316,7 @@ static int decode_audio(InputStream *ist, AVPacket *pkt, int *got_output)
                                               (AVRational){1, avctx->sample_rate});
     ist->nb_samples = decoded_frame->nb_samples;
     err = send_frame_to_filters(ist, decoded_frame);
-    decoded_frame->pts = AV_NOPTS_VALUE;
 
-fail:
     av_frame_unref(ist->filter_frame);
     av_frame_unref(decoded_frame);
     return err < 0 ? err : ret;
@@ -2217,7 +2325,7 @@ fail:
 static int decode_video(InputStream *ist, AVPacket *pkt, int *got_output, int eof)
 {
     AVFrame *decoded_frame;
-    int i, ret = 0, err = 0, resample_changed;
+    int i, ret = 0, err = 0;
     int64_t best_effort_timestamp;
     int64_t dts = AV_NOPTS_VALUE;
     AVPacket avpkt;
@@ -2332,39 +2440,6 @@ static int decode_video(InputStream *ist, AVPacket *pkt, int *got_output, int eo
     if (ist->st->sample_aspect_ratio.num)
         decoded_frame->sample_aspect_ratio = ist->st->sample_aspect_ratio;
 
-    resample_changed = ist->resample_width   != decoded_frame->width  ||
-                       ist->resample_height  != decoded_frame->height ||
-                       ist->resample_pix_fmt != decoded_frame->format;
-    if (resample_changed) {
-        av_log(NULL, AV_LOG_INFO,
-               "Input stream #%d:%d frame changed from size:%dx%d fmt:%s to size:%dx%d fmt:%s\n",
-               ist->file_index, ist->st->index,
-               ist->resample_width,  ist->resample_height,  av_get_pix_fmt_name(ist->resample_pix_fmt),
-               decoded_frame->width, decoded_frame->height, av_get_pix_fmt_name(decoded_frame->format));
-
-        ist->resample_width   = decoded_frame->width;
-        ist->resample_height  = decoded_frame->height;
-        ist->resample_pix_fmt = decoded_frame->format;
-
-        for (i = 0; i < ist->nb_filters; i++) {
-            err = ifilter_parameters_from_frame(ist->filters[i], decoded_frame);
-            if (err < 0) {
-                av_log(NULL, AV_LOG_ERROR,
-                       "Error reconfiguring input stream %d:%d filter %d\n",
-                       ist->file_index, ist->st->index, i);
-                goto fail;
-            }
-        }
-
-        for (i = 0; i < nb_filtergraphs; i++) {
-            if (ist_in_filtergraph(filtergraphs[i], ist) && ist->reinit_filters &&
-                configure_filtergraph(filtergraphs[i]) < 0) {
-                av_log(NULL, AV_LOG_FATAL, "Error reinitializing filters!\n");
-                exit_program(1);
-            }
-        }
-    }
-
     err = send_frame_to_filters(ist, decoded_frame);
 
 fail:
@@ -2434,11 +2509,18 @@ out:
 
 static int send_filter_eof(InputStream *ist)
 {
-    int i, ret;
+    int i, j, ret;
     for (i = 0; i < ist->nb_filters; i++) {
-        ret = av_buffersrc_add_frame(ist->filters[i]->filter, NULL);
-        if (ret < 0)
-            return ret;
+        if (ist->filters[i]->filter) {
+            ret = av_buffersrc_add_frame(ist->filters[i]->filter, NULL);
+            if (ret < 0)
+                return ret;
+        } else {
+            // the filtergraph was never configured
+            FilterGraph *fg = ist->filters[i]->graph;
+            for (j = 0; j < fg->nb_outputs; j++)
+                finish_output_stream(fg->outputs[j]->ost);
+        }
     }
     return 0;
 }
@@ -2544,6 +2626,9 @@ static int process_input_packet(InputStream *ist, const AVPacket *pkt, int no_eo
             break;
         }
 
+        if (got_output)
+            ist->got_output = 1;
+
         if (!got_output)
             break;
 
@@ -2721,17 +2806,9 @@ static int get_buffer(AVCodecContext *s, AVFrame *frame, int flags)
 
 static int init_input_stream(int ist_index, char *error, int error_len)
 {
-    int i, ret;
+    int ret;
     InputStream *ist = input_streams[ist_index];
 
-    for (i = 0; i < ist->nb_filters; i++) {
-        ret = ifilter_parameters_from_decoder(ist->filters[i], ist->dec_ctx);
-        if (ret < 0) {
-            av_log(NULL, AV_LOG_FATAL, "Error initializing filter input\n");
-            return ret;
-        }
-    }
-
     if (ist->decoding_needed) {
         AVCodec *codec = ist->dec;
         if (!codec) {
@@ -3125,17 +3202,6 @@ static int init_output_stream_encode(OutputStream *ost)
                 ost->st->disposition = AV_DISPOSITION_DEFAULT;
     }
 
-    if ((enc_ctx->codec_type == AVMEDIA_TYPE_VIDEO ||
-         enc_ctx->codec_type == AVMEDIA_TYPE_AUDIO) &&
-         filtergraph_is_simple(ost->filter->graph)) {
-            FilterGraph *fg = ost->filter->graph;
-
-            if (configure_filtergraph(fg)) {
-                av_log(NULL, AV_LOG_FATAL, "Error opening filters!\n");
-                exit_program(1);
-            }
-    }
-
     if (enc_ctx->codec_type == AVMEDIA_TYPE_VIDEO) {
         if (!ost->frame_rate.num)
             ost->frame_rate = av_buffersink_get_frame_rate(ost->filter->filter);
@@ -3475,10 +3541,6 @@ static int transcode_init(void)
         ost = output_streams[i];
 
         if (!ost->stream_copy) {
-#if CONFIG_LIBMFX
-            if (qsv_transcode_init(ost))
-                exit_program(1);
-#endif
 
 #if CONFIG_CUVID
             if (cuvid_transcode_init(ost))
@@ -3499,6 +3561,10 @@ static int transcode_init(void)
 
     /* open each encoder */
     for (i = 0; i < nb_output_streams; i++) {
+        // skip streams fed from filtergraphs until we have a frame for them
+        if (output_streams[i]->filter)
+            continue;
+
         ret = init_output_stream(output_streams[i], error, sizeof(error));
         if (ret < 0)
             goto dump_format;
@@ -3669,6 +3735,9 @@ static OutputStream *choose_output(void)
         if (ost->st->cur_dts == AV_NOPTS_VALUE)
             av_log(NULL, AV_LOG_DEBUG, "cur_dts is invalid (this is harmless if it occurs once at the start per stream)\n");
 
+        if (!ost->initialized && !ost->inputs_done)
+            return ost;
+
         if (!ost->finished && opts < opts_min) {
             opts_min = opts;
             ost_min  = ost->unavailable ? NULL : ost;
@@ -4326,7 +4395,7 @@ static int transcode_from_filter(FilterGraph *graph, InputStream **best_ist)
 static int transcode_step(void)
 {
     OutputStream *ost;
-    InputStream  *ist;
+    InputStream  *ist = NULL;
     int ret;
 
     ost = choose_output();
@@ -4340,11 +4409,34 @@ static int transcode_step(void)
         return AVERROR_EOF;
     }
 
-    if (ost->filter) {
+    if (ost->filter && !ost->filter->graph->graph) {
+        if (ifilter_has_all_input_formats(ost->filter->graph)) {
+            ret = configure_filtergraph(ost->filter->graph);
+            if (ret < 0) {
+                av_log(NULL, AV_LOG_ERROR, "Error reinitializing filters!\n");
+                return ret;
+            }
+        }
+    }
+
+    if (ost->filter && ost->filter->graph->graph) {
         if ((ret = transcode_from_filter(ost->filter->graph, &ist)) < 0)
             return ret;
         if (!ist)
             return 0;
+    } else if (ost->filter) {
+        int i;
+        for (i = 0; i < ost->filter->graph->nb_inputs; i++) {
+            InputFilter *ifilter = ost->filter->graph->inputs[i];
+            if (!ifilter->ist->got_output && !input_files[ifilter->ist->file_index]->eof_reached) {
+                ist = ifilter->ist;
+                break;
+            }
+        }
+        if (!ist) {
+            ost->inputs_done = 1;
+            return 0;
+        }
     } else {
         av_assert0(ost->source_index >= 0);
         ist = input_streams[ost->source_index];
diff --git a/ffmpeg.h b/ffmpeg.h
index ca35ccc260..56e35ebb9c 100644
--- a/ffmpeg.h
+++ b/ffmpeg.h
@@ -233,6 +233,9 @@ typedef struct InputFilter {
     struct InputStream *ist;
     struct FilterGraph *graph;
     uint8_t            *name;
+    enum AVMediaType    type;   // AVMEDIA_TYPE_SUBTITLE for sub2video
+
+    AVFifoBuffer *frame_queue;
 
     // parameters configured for this input
     int format;
@@ -321,14 +324,6 @@ typedef struct InputStream {
     int guess_layout_max;
 
     int autorotate;
-    int resample_height;
-    int resample_width;
-    int resample_pix_fmt;
-
-    int      resample_sample_fmt;
-    int      resample_sample_rate;
-    int      resample_channels;
-    uint64_t resample_channel_layout;
 
     int fix_sub_duration;
     struct { /* previous decoded subtitle and related variables */
@@ -379,6 +374,8 @@ typedef struct InputStream {
 
     int64_t *dts_buffer;
     int nb_dts_buffer;
+
+    int got_output;
 } InputStream;
 
 typedef struct InputFile {
@@ -507,6 +504,8 @@ typedef struct OutputStream {
     // parameters are set in the AVStream.
     int initialized;
 
+    int inputs_done;
+
     const char *attachment_filename;
     int copy_initial_nonkeyframes;
     int copy_prior_start;
@@ -636,7 +635,6 @@ int init_simple_filtergraph(InputStream *ist, OutputStream *ost);
 int init_complex_filtergraph(FilterGraph *fg);
 
 int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame);
-int ifilter_parameters_from_decoder(InputFilter *ifilter, const AVCodecContext *avctx);
 
 int ffmpeg_parse_options(int argc, char **argv);
 
@@ -645,7 +643,6 @@ int dxva2_init(AVCodecContext *s);
 int vda_init(AVCodecContext *s);
 int videotoolbox_init(AVCodecContext *s);
 int qsv_init(AVCodecContext *s);
-int qsv_transcode_init(OutputStream *ost);
 int vaapi_decode_init(AVCodecContext *avctx);
 int vaapi_device_init(const char *device);
 int cuvid_init(AVCodecContext *s);
diff --git a/ffmpeg_cuvid.c b/ffmpeg_cuvid.c
index baf6eee9f8..46540795d4 100644
--- a/ffmpeg_cuvid.c
+++ b/ffmpeg_cuvid.c
@@ -125,7 +125,6 @@ int cuvid_transcode_init(OutputStream *ost)
         }
 
         ist->hwaccel_ctx = ctx;
-        ist->resample_pix_fmt = AV_PIX_FMT_CUDA;
         ist->hwaccel_uninit = cuvid_uninit;
 
         /* This is a bit hacky, av_hwframe_ctx_init is called by the cuvid decoder
diff --git a/ffmpeg_filter.c b/ffmpeg_filter.c
index f13f523045..8490f4a455 100644
--- a/ffmpeg_filter.c
+++ b/ffmpeg_filter.c
@@ -217,6 +217,10 @@ int init_simple_filtergraph(InputStream *ist, OutputStream *ost)
     fg->inputs[0]->graph = fg;
     fg->inputs[0]->format = -1;
 
+    fg->inputs[0]->frame_queue = av_fifo_alloc(8 * sizeof(AVFrame*));
+    if (!fg->inputs[0]->frame_queue)
+        exit_program(1);
+
     GROW_ARRAY(ist->filters, ist->nb_filters);
     ist->filters[ist->nb_filters - 1] = fg->inputs[0];
 
@@ -295,6 +299,11 @@ static void init_input_filter(FilterGraph *fg, AVFilterInOut *in)
     fg->inputs[fg->nb_inputs - 1]->ist   = ist;
     fg->inputs[fg->nb_inputs - 1]->graph = fg;
     fg->inputs[fg->nb_inputs - 1]->format = -1;
+    fg->inputs[fg->nb_inputs - 1]->type = ist->st->codecpar->codec_type;
+
+    fg->inputs[fg->nb_inputs - 1]->frame_queue = av_fifo_alloc(8 * sizeof(AVFrame*));
+    if (!fg->inputs[fg->nb_inputs - 1]->frame_queue)
+        exit_program(1);
 
     GROW_ARRAY(ist->filters, ist->nb_filters);
     ist->filters[ist->nb_filters - 1] = fg->inputs[fg->nb_inputs - 1];
@@ -691,12 +700,15 @@ static int sub2video_prepare(InputStream *ist, InputFilter *ifilter)
         }
         av_log(avf, AV_LOG_INFO, "sub2video: using %dx%d canvas\n", w, h);
     }
-    ist->sub2video.w = ist->resample_width  = ifilter->width  = w;
-    ist->sub2video.h = ist->resample_height = ifilter->height = h;
+    ist->sub2video.w = ifilter->width  = w;
+    ist->sub2video.h = ifilter->height = h;
+
+    ifilter->width  = ist->dec_ctx->width  ? ist->dec_ctx->width  : ist->sub2video.w;
+    ifilter->height = ist->dec_ctx->height ? ist->dec_ctx->height : ist->sub2video.h;
 
     /* rectangles are AV_PIX_FMT_PAL8, but we have no guarantee that the
        palettes for all rectangles are identical or compatible */
-    ist->resample_pix_fmt = ifilter->format = AV_PIX_FMT_RGB32;
+    ifilter->format = AV_PIX_FMT_RGB32;
 
     ist->sub2video.frame = av_frame_alloc();
     if (!ist->sub2video.frame)
@@ -1133,36 +1145,6 @@ int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame)
     return 0;
 }
 
-int ifilter_parameters_from_decoder(InputFilter *ifilter, const AVCodecContext *avctx)
-{
-    av_buffer_unref(&ifilter->hw_frames_ctx);
-
-    if (avctx->codec_type == AVMEDIA_TYPE_VIDEO)
-        ifilter->format = avctx->pix_fmt;
-    else
-        ifilter->format = avctx->sample_fmt;
-
-    ifilter->width               = avctx->width;
-    ifilter->height              = avctx->height;
-    if (ifilter->ist && ifilter->ist->st && ifilter->ist->st->sample_aspect_ratio.num)
-        ifilter->sample_aspect_ratio = ifilter->ist->st->sample_aspect_ratio;
-    else
-        ifilter->sample_aspect_ratio = avctx->sample_aspect_ratio;
-
-    ifilter->sample_rate         = avctx->sample_rate;
-    ifilter->channels            = avctx->channels;
-    ifilter->channel_layout      = avctx->channel_layout;
-
-    if (ifilter->ist && ifilter->ist->hw_frames_ctx) {
-        ifilter->format = ifilter->ist->resample_pix_fmt;
-        ifilter->hw_frames_ctx = av_buffer_ref(ifilter->ist->hw_frames_ctx);
-        if (!ifilter->hw_frames_ctx)
-            return AVERROR(ENOMEM);
-    }
-
-    return 0;
-}
-
 int ist_in_filtergraph(FilterGraph *fg, InputStream *ist)
 {
     int i;
diff --git a/ffmpeg_opt.c b/ffmpeg_opt.c
index 6a47d32b53..e2c0176e14 100644
--- a/ffmpeg_opt.c
+++ b/ffmpeg_opt.c
@@ -736,10 +736,6 @@ static void add_input_streams(OptionsContext *o, AVFormatContext *ic)
             // avformat_find_stream_info() doesn't set this for us anymore.
             ist->dec_ctx->framerate = st->avg_frame_rate;
 
-            ist->resample_height  = ist->dec_ctx->height;
-            ist->resample_width   = ist->dec_ctx->width;
-            ist->resample_pix_fmt = ist->dec_ctx->pix_fmt;
-
             MATCH_PER_STREAM_OPT(frame_rates, str, framerate, ic, st);
             if (framerate && av_parse_video_rate(&ist->framerate,
                                                  framerate) < 0) {
@@ -804,12 +800,6 @@ static void add_input_streams(OptionsContext *o, AVFormatContext *ic)
             ist->guess_layout_max = INT_MAX;
             MATCH_PER_STREAM_OPT(guess_layout_max, i, ist->guess_layout_max, ic, st);
             guess_input_channel_layout(ist);
-
-            ist->resample_sample_fmt     = ist->dec_ctx->sample_fmt;
-            ist->resample_sample_rate    = ist->dec_ctx->sample_rate;
-            ist->resample_channels       = ist->dec_ctx->channels;
-            ist->resample_channel_layout = ist->dec_ctx->channel_layout;
-
             break;
         case AVMEDIA_TYPE_DATA:
         case AVMEDIA_TYPE_SUBTITLE: {
@@ -2021,33 +2011,6 @@ static int init_complex_filters(void)
     return 0;
 }
 
-static int configure_complex_filters(void)
-{
-    int i, j, ret = 0;
-
-    for (i = 0; i < nb_filtergraphs; i++) {
-        FilterGraph *fg = filtergraphs[i];
-
-        if (filtergraph_is_simple(fg))
-            continue;
-
-        for (j = 0; j < fg->nb_inputs; j++) {
-            ret = ifilter_parameters_from_decoder(fg->inputs[j],
-                                                  fg->inputs[j]->ist->dec_ctx);
-            if (ret < 0) {
-                av_log(NULL, AV_LOG_ERROR,
-                       "Error initializing filtergraph %d input %d\n", i, j);
-                return ret;
-            }
-        }
-
-        ret = configure_filtergraph(filtergraphs[i]);
-        if (ret < 0)
-            return ret;
-    }
-    return 0;
-}
-
 static int open_output_file(OptionsContext *o, const char *filename)
 {
     AVFormatContext *oc;
@@ -3291,13 +3254,6 @@ int ffmpeg_parse_options(int argc, char **argv)
         goto fail;
     }
 
-    /* configure the complex filtergraphs */
-    ret = configure_complex_filters();
-    if (ret < 0) {
-        av_log(NULL, AV_LOG_FATAL, "Error configuring complex filters.\n");
-        goto fail;
-    }
-
 fail:
     uninit_parse_context(&octx);
     if (ret < 0) {
diff --git a/ffmpeg_qsv.c b/ffmpeg_qsv.c
index 86824b60f2..7442750029 100644
--- a/ffmpeg_qsv.c
+++ b/ffmpeg_qsv.c
@@ -107,85 +107,3 @@ int qsv_init(AVCodecContext *s)
 
     return 0;
 }
-
-int qsv_transcode_init(OutputStream *ost)
-{
-    InputStream *ist;
-    const enum AVPixelFormat *pix_fmt;
-
-    int err, i;
-    AVBufferRef *encode_frames_ref = NULL;
-    AVHWFramesContext *encode_frames;
-    AVQSVFramesContext *qsv_frames;
-
-    /* check if the encoder supports QSV */
-    if (!ost->enc->pix_fmts)
-        return 0;
-    for (pix_fmt = ost->enc->pix_fmts; *pix_fmt != AV_PIX_FMT_NONE; pix_fmt++)
-        if (*pix_fmt == AV_PIX_FMT_QSV)
-            break;
-    if (*pix_fmt == AV_PIX_FMT_NONE)
-        return 0;
-
-    if (strcmp(ost->avfilter, "null") || ost->source_index < 0)
-        return 0;
-
-    /* check if the decoder supports QSV and the output only goes to this stream */
-    ist = input_streams[ost->source_index];
-    if (ist->hwaccel_id != HWACCEL_QSV || !ist->dec || !ist->dec->pix_fmts)
-        return 0;
-    for (pix_fmt = ist->dec->pix_fmts; *pix_fmt != AV_PIX_FMT_NONE; pix_fmt++)
-        if (*pix_fmt == AV_PIX_FMT_QSV)
-            break;
-    if (*pix_fmt == AV_PIX_FMT_NONE)
-        return 0;
-
-    for (i = 0; i < nb_output_streams; i++)
-        if (output_streams[i] != ost &&
-            output_streams[i]->source_index == ost->source_index)
-            return 0;
-
-    av_log(NULL, AV_LOG_VERBOSE, "Setting up QSV transcoding\n");
-
-    if (!hw_device_ctx) {
-        err = qsv_device_init(ist);
-        if (err < 0)
-            goto fail;
-    }
-
-    // This creates a dummy hw_frames_ctx for the encoder to be
-    // suitably initialised.  It only contains one real frame, so
-    // hopefully doesn't waste too much memory.
-
-    encode_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx);
-    if (!encode_frames_ref) {
-        err = AVERROR(ENOMEM);
-        goto fail;
-    }
-    encode_frames = (AVHWFramesContext*)encode_frames_ref->data;
-    qsv_frames = encode_frames->hwctx;
-
-    encode_frames->width     = FFALIGN(ist->resample_width,  32);
-    encode_frames->height    = FFALIGN(ist->resample_height, 32);
-    encode_frames->format    = AV_PIX_FMT_QSV;
-    encode_frames->sw_format = AV_PIX_FMT_NV12;
-    encode_frames->initial_pool_size = 1;
-
-    qsv_frames->frame_type = MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
-
-    err = av_hwframe_ctx_init(encode_frames_ref);
-    if (err < 0)
-        goto fail;
-
-    ist->dec_ctx->pix_fmt       = AV_PIX_FMT_QSV;
-    ist->resample_pix_fmt       = AV_PIX_FMT_QSV;
-
-    ost->enc_ctx->pix_fmt       = AV_PIX_FMT_QSV;
-    ost->enc_ctx->hw_frames_ctx = encode_frames_ref;
-
-    return 0;
-
-fail:
-    av_buffer_unref(&encode_frames_ref);
-    return err;
-}
diff --git a/tests/ref/fate/exr-rgb-scanline-pxr24-half-uint32-13x9 b/tests/ref/fate/exr-rgb-scanline-pxr24-half-uint32-13x9
index 4a54c336e3..e2aa9efb9f 100644
--- a/tests/ref/fate/exr-rgb-scanline-pxr24-half-uint32-13x9
+++ b/tests/ref/fate/exr-rgb-scanline-pxr24-half-uint32-13x9
@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 13x9
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      936, 0x7b44246e
diff --git a/tests/ref/fate/exr-rgba-scanline-float-half-b44-12x8-l1 b/tests/ref/fate/exr-rgba-scanline-float-half-b44-12x8-l1
index c84d388fff..cd25f618f4 100644
--- a/tests/ref/fate/exr-rgba-scanline-float-half-b44-12x8-l1
+++ b/tests/ref/fate/exr-rgba-scanline-float-half-b44-12x8-l1
@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 12x8
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      768, 0x1de5c7f1
diff --git a/tests/ref/fate/exr-rgba-scanline-float-half-b44-12x8-l2 b/tests/ref/fate/exr-rgba-scanline-float-half-b44-12x8-l2
index 1dd42d9175..ecfba85e83 100644
--- a/tests/ref/fate/exr-rgba-scanline-float-half-b44-12x8-l2
+++ b/tests/ref/fate/exr-rgba-scanline-float-half-b44-12x8-l2
@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 12x8
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      768, 0xe08ca6d3
diff --git a/tests/ref/fate/exr-rgba-scanline-float-half-b44-13x9-l1 b/tests/ref/fate/exr-rgba-scanline-float-half-b44-13x9-l1
index 595c721d50..3e82e2aaa4 100644
--- a/tests/ref/fate/exr-rgba-scanline-float-half-b44-13x9-l1
+++ b/tests/ref/fate/exr-rgba-scanline-float-half-b44-13x9-l1
@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 13x9
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      936, 0xdcb42186
diff --git a/tests/ref/fate/exr-rgba-scanline-float-half-b44-13x9-l2 b/tests/ref/fate/exr-rgba-scanline-float-half-b44-13x9-l2
index 079d1d877f..f237d17d3f 100644
--- a/tests/ref/fate/exr-rgba-scanline-float-half-b44-13x9-l2
+++ b/tests/ref/fate/exr-rgba-scanline-float-half-b44-13x9-l2
@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 13x9
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      936, 0x7f710bf5
diff --git a/tests/ref/fate/exr-rgba-scanline-float-half-b44a-12x8-l1 b/tests/ref/fate/exr-rgba-scanline-float-half-b44a-12x8-l1
index 6d202af808..43313e3712 100644
--- a/tests/ref/fate/exr-rgba-scanline-float-half-b44a-12x8-l1
+++ b/tests/ref/fate/exr-rgba-scanline-float-half-b44a-12x8-l1
@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 12x8
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      768, 0xe200c160
diff --git a/tests/ref/fate/exr-rgba-scanline-float-half-b44a-12x8-l2 b/tests/ref/fate/exr-rgba-scanline-float-half-b44a-12x8-l2
index 1dd42d9175..ecfba85e83 100644
--- a/tests/ref/fate/exr-rgba-scanline-float-half-b44a-12x8-l2
+++ b/tests/ref/fate/exr-rgba-scanline-float-half-b44a-12x8-l2
@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 12x8
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      768, 0xe08ca6d3
diff --git a/tests/ref/fate/exr-rgba-scanline-float-half-b44a-13x9-l1 b/tests/ref/fate/exr-rgba-scanline-float-half-b44a-13x9-l1
index 9fcf309368..e43c31ebd5 100644
--- a/tests/ref/fate/exr-rgba-scanline-float-half-b44a-13x9-l1
+++ b/tests/ref/fate/exr-rgba-scanline-float-half-b44a-13x9-l1
@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 13x9
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      936, 0x911718ac
diff --git a/tests/ref/fate/exr-rgba-scanline-float-half-b44a-13x9-l2 b/tests/ref/fate/exr-rgba-scanline-float-half-b44a-13x9-l2
index 079d1d877f..f237d17d3f 100644
--- a/tests/ref/fate/exr-rgba-scanline-float-half-b44a-13x9-l2
+++ b/tests/ref/fate/exr-rgba-scanline-float-half-b44a-13x9-l2
@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 13x9
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      936, 0x7f710bf5
diff --git a/tests/ref/fate/vp8-size-change b/tests/ref/fate/vp8-size-change
index 741b6d65fb..5105bc6f55 100644
--- a/tests/ref/fate/vp8-size-change
+++ b/tests/ref/fate/vp8-size-change
@@ -4,36 +4,36 @@
 #tb 0: 1/30
 #media_type 0: video
 #codec_id 0: rawvideo
-#dimensions 0: 1920x1080
+#dimensions 0: 160x90
 #sar 0: 1/1
 #stream#, dts,        pts, duration,     size, hash
-0,          0,          0,        1,  3110400, 7dde8cd136ab4b04a95d9856b941697e
-0,          1,          1,        1,  3110400, aa885f78cb6374b5bfcc66a4fc57026f
-0,          2,          2,        1,  3110400, b69b7b56f549a3f9b0a603940bac85ed
-0,          3,          3,        1,  3110400, 20e2e0f0c89c58828b6a3b10d9e175e5
-0,          4,          4,        1,  3110400, 483997936e7d6bb849e64d50426ec689
-0,          5,          5,        1,  3110400, c85ef97a853ff594e2bfdf0a0a581dcc
-0,          6,          6,        1,  3110400, c5e7b9ff4c25391793446da788cb83a9
-0,          7,          7,        1,  3110400, 63f93e89d24162e2f6328edbc6716b33
-0,          8,          8,        1,  3110400, 0e346ab4831ce8c69001153c72b7b827
-0,          9,          9,        1,  3110400, c526c21511d8bec2659d3d43d93734f2
-0,         10,         10,        1,  3110400, e95d01d5f9fb81a98bd34305c7ab30f8
-0,         11,         11,        1,  3110400, 177e75e7516e8746d31b43ea9d39e6b1
-0,         12,         12,        1,  3110400, 489d2bc0da93f118dc9a2697275697a7
-0,         13,         13,        1,  3110400, a2dc00d627350ff1ab302bcbad5ca5ac
-0,         14,         14,        1,  3110400, 20ce143831b0189f763ee5bee9c51188
-0,         15,         15,        1,  3110400, 7822fd908bd81b521c23fa4a639caf9e
-0,         16,         16,        1,  3110400, dabc4febbe09734126ac6f5a5180ba8c
-0,         17,         17,        1,  3110400, ef88f0d6667feefac1471b065208e1c8
-0,         18,         18,        1,  3110400, 7c7fc665a6fd9e19af9358bbdc162a51
-0,         19,         19,        1,  3110400, f2bcf32f734f99506bdd0a0376badf82
-0,         20,         20,        1,  3110400, 06809c2d277fd3b3918ebb4b65c27661
-0,         21,         21,        1,  3110400, e403e9e86fa5d519f65c565b3add84b5
-0,         22,         22,        1,  3110400, d2b876730e12245cacb578307794349a
-0,         23,         23,        1,  3110400, dfdfd8cb626a96138f6a2c1953dcf5ec
-0,         24,         24,        1,  3110400, 0ac58c28575b804d9e63395653c3aef2
-0,         25,         25,        1,  3110400, 641f2a78e338c733ef159bd36ec7966f
-0,         26,         26,        1,  3110400, 9402d455fa5bd556b85f479c42c3a4d2
-0,         27,         27,        1,  3110400, 0044d42b4048bc93112aa59789dbdc2d
-0,         28,         28,        1,  3110400, 5d9e5c5ba35f6f452e5f31ccff9e819c
-0,         29,         29,        1,  3110400, 307a55a94739b4cfdf41f7da7e5c0135
+0,          0,          0,        1,    21600, 5abd6c163522c7e882f7e9c369293bf9
+0,          1,          1,        1,    21600, 5c11d8cc9cc9102d0ef5afd1dc64aff1
+0,          2,          2,        1,    21600, cbeffa9ea9d682af77d3fd0fdf12c8c1
+0,          3,          3,        1,    21600, ea7cec515fcf8ccbc932d9e9b472cdc7
+0,          4,          4,        1,    21600, 23038b711dbac95ce710657b1fef5901
+0,          5,          5,        1,    21600, e0d6fb46bb5c0d939ee33af82b887668
+0,          6,          6,        1,    21600, 694518f14d3a2bd3c319bc0b098c78bb
+0,          7,          7,        1,    21600, c1c7394bd4236afbc773af52ef7a10ea
+0,          8,          8,        1,    21600, 4d8d3b2c9a637f963521585ea879357b
+0,          9,          9,        1,    21600, b4444dc3cbf1b6cdd8047d3dcd497ffd
+0,         10,         10,        1,    21600, 65e5d667ec9ceb636e21357f032ce800
+0,         11,         11,        1,    21600, fd9a4c67598051074387b640df7edaa9
+0,         12,         12,        1,    21600, 0e54e22d90f6296ae6989c83846272cd
+0,         13,         13,        1,    21600, db4b1727450243b202bfec5ed6c73ae0
+0,         14,         14,        1,    21600, ab37a84be075ca42cc7351ff9fb1cb47
+0,         15,         15,        1,    21600, ae4d2d297e646bd8e05e76b457d9b576
+0,         16,         16,        1,    21600, e7cfd580e3c3d7c3f2f5136d1e548595
+0,         17,         17,        1,    21600, cbec09314a0b7ad53f4893eb474e1c65
+0,         18,         18,        1,    21600, e1fa89cd63c37496bc86f18694324d88
+0,         19,         19,        1,    21600, e9655b151253950313810228278ca104
+0,         20,         20,        1,    21600, 69ba31c0eff7bc93f4180173d8e64c60
+0,         21,         21,        1,    21600, 368a1f6a1172d7d56f695153b234a330
+0,         22,         22,        1,    21600, 6c298b196e16c64f7c2f407ba1242937
+0,         23,         23,        1,    21600, bf54474112ed5592c4d890e3313881a0
+0,         24,         24,        1,    21600, 945d49abedb0606b6a009c8b5d8face3
+0,         25,         25,        1,    21600, dd6ebef7b6f24619910de811918d3437
+0,         26,         26,        1,    21600, 7952346fc0f1eff3914e0d7646b3cf28
+0,         27,         27,        1,    21600, 26bd0d6b21e8a2df17af8d1446fba745
+0,         28,         28,        1,    21600, b0d91600416716d81c1f73ac141a0b62
+0,         29,         29,        1,    21600, 08f16698beb9cc15f7115961bd69e995
diff --git a/tests/ref/lavf/ffm b/tests/ref/lavf/ffm
index 54c56034aa..d9fa8d52cb 100644
--- a/tests/ref/lavf/ffm
+++ b/tests/ref/lavf/ffm
@@ -1,3 +1,3 @@
-a0e9616f0d9a8c1029f3220b1b9175f4 *./tests/data/lavf/lavf.ffm
+ca2a450cd0d1e299514a345923b4c82a *./tests/data/lavf/lavf.ffm
 376832 ./tests/data/lavf/lavf.ffm
 ./tests/data/lavf/lavf.ffm CRC=0x000e23ae
diff --git a/tests/ref/lavf/mxf b/tests/ref/lavf/mxf
index e1c0c79b05..103ef010b7 100644
--- a/tests/ref/lavf/mxf
+++ b/tests/ref/lavf/mxf
@@ -1,9 +1,9 @@
-f9b570c7b4fbbc2b71f2236b32e7cbb6 *./tests/data/lavf/lavf.mxf
+c61e677b4facc407ca82b1d7c6298484 *./tests/data/lavf/lavf.mxf
 525369 ./tests/data/lavf/lavf.mxf
 ./tests/data/lavf/lavf.mxf CRC=0xdbfff6f1
-8f6a9a6b409f0f5a0bf003f8dea26314 *./tests/data/lavf/lavf.mxf
+1ae017b5879992431614b1c7ab3b48c5 *./tests/data/lavf/lavf.mxf
 560697 ./tests/data/lavf/lavf.mxf
 ./tests/data/lavf/lavf.mxf CRC=0x11a6178e
-10ac0f158fc0af356439b818de7601e3 *./tests/data/lavf/lavf.mxf
+27fac91dce279630a04b94b93518a9d8 *./tests/data/lavf/lavf.mxf
 525369 ./tests/data/lavf/lavf.mxf
 ./tests/data/lavf/lavf.mxf CRC=0xdbfff6f1