Merge commit 'f03f78bc1c99b1e29624418e2f7315b8a47981e9'

* commit 'f03f78bc1c99b1e29624418e2f7315b8a47981e9':
  mpegvideo_enc: handle encoding errors with b_strategy=2

Merged-by: Clément Bœsch <u@pkh.me>

1 files changed, 18 insertions, 0 deletions

diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index 8051605ddd..974955493a 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -1453,6 +1453,7 @@ static int estimate_best_b_count(MpegEncContext *s)
     int i, j, out_size, p_lambda, b_lambda, lambda2;
     int64_t best_rd  = INT64_MAX;
     int best_b_count = -1;
+    int ret = 0;
 
     if (!c)
         return AVERROR(ENOMEM);
@@ -1528,6 +1529,10 @@ static int estimate_best_b_count(MpegEncContext *s)
         s->tmp_frames[0]->quality   = 1 * FF_QP2LAMBDA;
 
         out_size = encode_frame(c, s->tmp_frames[0]);
+        if (out_size < 0) {
+            ret = out_size;
+            goto fail;
+        }
 
         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
 
@@ -1539,6 +1544,10 @@ static int estimate_best_b_count(MpegEncContext *s)
             s->tmp_frames[i + 1]->quality   = is_p ? p_lambda : b_lambda;
 
             out_size = encode_frame(c, s->tmp_frames[i + 1]);
+            if (out_size < 0) {
+                ret = out_size;
+                goto fail;
+            }
 
             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
         }
@@ -1546,6 +1555,10 @@ static int estimate_best_b_count(MpegEncContext *s)
         /* get the delayed frames */
         while (out_size) {
             out_size = encode_frame(c, NULL);
+            if (out_size < 0) {
+                ret = out_size;
+                goto fail;
+            }
             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
         }
 
@@ -1560,6 +1573,9 @@ static int estimate_best_b_count(MpegEncContext *s)
     avcodec_free_context(&c);
 
     return best_b_count;
+fail:
+    avcodec_free_context(&c);
+    return ret;
 }
 
 static int select_input_picture(MpegEncContext *s)
@@ -1640,6 +1656,8 @@ static int select_input_picture(MpegEncContext *s)
                 }
             } else if (s->b_frame_strategy == 2) {
                 b_frames = estimate_best_b_count(s);
+                if (b_frames < 0)
+                    return b_frames;
             }
 
             emms_c();