h264: fix ff_generate_sliding_window_mmcos() prototype.

It's been returning an error value since
bad446e251405dc250c3cbee199072e083a1e4b9

Also check for the errors it returns.
(cherry picked from commit ea382767ad2191acbe97e90624059723e15f0e4b)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

3 files changed, 13 insertions, 7 deletions

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index c8c3974ee7..e28cc636b4 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -2360,7 +2360,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
     MpegEncContext *const s0 = &h0->s;
     unsigned int first_mb_in_slice;
     unsigned int pps_id;
-    int num_ref_idx_active_override_flag;
+    int num_ref_idx_active_override_flag, ret;
     unsigned int slice_type, tmp, i, j;
     int default_ref_list_done = 0;
     int last_pic_structure, last_pic_dropable;
@@ -2795,7 +2795,9 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
             s->current_picture_ptr->frame_num = h->prev_frame_num;
             ff_thread_report_progress(&s->current_picture_ptr->f, INT_MAX, 0);
             ff_thread_report_progress(&s->current_picture_ptr->f, INT_MAX, 1);
-            ff_generate_sliding_window_mmcos(h, 1);
+            if ((ret = ff_generate_sliding_window_mmcos(h, 1)) < 0 &&
+                s->avctx->err_recognition & AV_EF_EXPLODE)
+                return ret;
             if (ff_h264_execute_ref_pic_marking(h, h->mmco, h->mmco_index) < 0 &&
                 (s->avctx->err_recognition & AV_EF_EXPLODE))
                 return AVERROR_INVALIDDATA;
diff --git a/libavcodec/h264.h b/libavcodec/h264.h
index c3052bb9ed..65d1fd8c15 100644
--- a/libavcodec/h264.h
+++ b/libavcodec/h264.h
@@ -671,7 +671,7 @@ int ff_h264_execute_ref_pic_marking(H264Context *h, MMCO *mmco, int mmco_count);
 int ff_h264_decode_ref_pic_marking(H264Context *h, GetBitContext *gb,
                                    int first_slice);
 
-void ff_generate_sliding_window_mmcos(H264Context *h, int first_slice);
+int ff_generate_sliding_window_mmcos(H264Context *h, int first_slice);
 
 /**
  * Check if the top & left blocks are available if needed & change the
diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 46e11ebfc5..20058c2e0c 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -492,7 +492,7 @@ static int check_opcodes(MMCO *mmco1, MMCO *mmco2, int n_mmcos)
     return 0;
 }
 
-void ff_generate_sliding_window_mmcos(H264Context *h, int first_slice)
+int ff_generate_sliding_window_mmcos(H264Context *h, int first_slice)
 {
     MpegEncContext * const s = &h->s;
     MMCO mmco_temp[MAX_MMCO_COUNT], *mmco = first_slice ? h->mmco : mmco_temp;
@@ -523,6 +523,7 @@ void ff_generate_sliding_window_mmcos(H264Context *h, int first_slice)
                mmco_index, h->mmco_index, i);
         return AVERROR_INVALIDDATA;
     }
+    return 0;
 }
 
 int ff_h264_execute_ref_pic_marking(H264Context *h, MMCO *mmco, int mmco_count){
@@ -696,7 +697,7 @@ int ff_h264_decode_ref_pic_marking(H264Context *h, GetBitContext *gb,
                                    int first_slice)
 {
     MpegEncContext * const s = &h->s;
-    int i;
+    int i, ret;
     MMCO mmco_temp[MAX_MMCO_COUNT], *mmco = first_slice ? h->mmco : mmco_temp;
     int mmco_index = 0;
 
@@ -753,8 +754,11 @@ int ff_h264_decode_ref_pic_marking(H264Context *h, GetBitContext *gb,
             }
             mmco_index = i;
         } else {
-            if (first_slice)
-                ff_generate_sliding_window_mmcos(h, first_slice);
+            if (first_slice) {
+                ret = ff_generate_sliding_window_mmcos(h, first_slice);
+                if (ret < 0 && s->avctx->err_recognition & AV_EF_EXPLODE)
+                    return ret;
+            }
             mmco_index = -1;
         }
     }