avfilter/all: propagate errors of functions from avfilter/formats

Many of the functions from avfilter/formats can return errors, usually AVERROR(ENOMEM).
This propagates the return values.

All of these were found by using av_warn_unused_result, demonstrating its utility.

Tested with FATE. I am least sure of the changes to avfilter/filtergraph,
since I don't know what/how reduce_format is intended to behave and how it should
react to errors.

Fixes: CID 1325680, 1325679, 1325678.

Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Previous version Reviewed-by: Nicolas George <george@nsup.org>
Previous version Reviewed-by: Clément Bœsch <u@pkh.me>
Signed-off-by: Ganesh Ajjanagadde <gajjanagadde@gmail.com>

1 files changed, 9 insertions, 6 deletions

diff --git a/libavfilter/vf_mergeplanes.c b/libavfilter/vf_mergeplanes.c
index ea796be10f..2d9db843f4 100644
--- a/libavfilter/vf_mergeplanes.c
+++ b/libavfilter/vf_mergeplanes.c
@@ -116,22 +116,25 @@ static int query_formats(AVFilterContext *ctx)
 {
     MergePlanesContext *s = ctx->priv;
     AVFilterFormats *formats = NULL;
-    int i;
+    int i, ret;
 
     s->outdesc = av_pix_fmt_desc_get(s->out_fmt);
     for (i = 0; av_pix_fmt_desc_get(i); i++) {
         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(i);
         if (desc->comp[0].depth == s->outdesc->comp[0].depth &&
-            av_pix_fmt_count_planes(i) == desc->nb_components)
-            ff_add_format(&formats, i);
+            av_pix_fmt_count_planes(i) == desc->nb_components &&
+            (ret = ff_add_format(&formats, i)) < 0)
+                return ret;
     }
 
     for (i = 0; i < s->nb_inputs; i++)
-        ff_formats_ref(formats, &ctx->inputs[i]->out_formats);
+        if ((ret = ff_formats_ref(formats, &ctx->inputs[i]->out_formats)) < 0)
+            return ret;
 
     formats = NULL;
-    ff_add_format(&formats, s->out_fmt);
-    ff_formats_ref(formats, &ctx->outputs[0]->in_formats);
+    if ((ret = ff_add_format(&formats, s->out_fmt)) < 0 ||
+        (ret = ff_formats_ref(formats, &ctx->outputs[0]->in_formats)) < 0)
+        return ret;
 
     return 0;
 }