avcodec/vp56: Check for the bitstream end, pass error codes on

Fixes timeout
Fixes: 446/fuzz-3-ffmpeg_VIDEO_AV_CODEC_ID_VP6_fuzzer

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 9e6a2427558a718be0c1fffacffd935f630a7a8d)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

4 files changed, 30 insertions, 11 deletions

diff --git a/libavcodec/vp5.c b/libavcodec/vp5.c
index 6c664c6693..aba177c89f 100644
--- a/libavcodec/vp5.c
+++ b/libavcodec/vp5.c
@@ -170,7 +170,7 @@ static int vp5_parse_coeff_models(VP56Context *s)
     return 0;
 }
 
-static void vp5_parse_coeff(VP56Context *s)
+static int vp5_parse_coeff(VP56Context *s)
 {
     VP56RangeCoder *c = &s->c;
     VP56Model *model = s->modelp;
@@ -180,6 +180,11 @@ static void vp5_parse_coeff(VP56Context *s)
     int b, i, cg, idx, ctx, ctx_last;
     int pt = 0;    /* plane type (0 for Y, 1 for U or V) */
 
+    if (c->end >= c->buffer && c->bits >= 0) {
+        av_log(s->avctx, AV_LOG_ERROR, "End of AC stream reached in vp5_parse_coeff\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     for (b=0; b<6; b++) {
         int ct = 1;    /* code type */
 
@@ -245,6 +250,7 @@ static void vp5_parse_coeff(VP56Context *s)
                 s->coeff_ctx[ff_vp56_b6to4[b]][i] = 5;
         s->above_blocks[s->above_block_idx[b]].not_null_dc = s->coeff_ctx[ff_vp56_b6to4[b]][0];
     }
+    return 0;
 }
 
 static void vp5_default_models_init(VP56Context *s)
diff --git a/libavcodec/vp56.c b/libavcodec/vp56.c
index 631924828d..d8fe994b8c 100644
--- a/libavcodec/vp56.c
+++ b/libavcodec/vp56.c
@@ -381,12 +381,13 @@ static void vp56_mc(VP56Context *s, int b, int plane, uint8_t *src,
     }
 }
 
-static void vp56_decode_mb(VP56Context *s, int row, int col, int is_alpha)
+static int vp56_decode_mb(VP56Context *s, int row, int col, int is_alpha)
 {
     AVFrame *frame_current, *frame_ref;
     VP56mb mb_type;
     VP56Frame ref_frame;
     int b, ab, b_max, plane, off;
+    int ret;
 
     if (s->frames[VP56_FRAME_CURRENT]->key_frame)
         mb_type = VP56_MB_INTRA;
@@ -394,14 +395,16 @@ static void vp56_decode_mb(VP56Context *s, int row, int col, int is_alpha)
         mb_type = vp56_decode_mv(s, row, col);
     ref_frame = ff_vp56_reference_frame[mb_type];
 
-    s->parse_coeff(s);
+    ret = s->parse_coeff(s);
+    if (ret < 0)
+        return ret;
 
     vp56_add_predictors_dc(s, ref_frame);
 
     frame_current = s->frames[VP56_FRAME_CURRENT];
     frame_ref = s->frames[ref_frame];
     if (mb_type != VP56_MB_INTRA && !frame_ref->data[0])
-        return;
+        return 0;
 
     ab = 6*is_alpha;
     b_max = 6 - 2*is_alpha;
@@ -451,6 +454,7 @@ static void vp56_decode_mb(VP56Context *s, int row, int col, int is_alpha)
         s->block_coeff[4][0] = 0;
         s->block_coeff[5][0] = 0;
     }
+    return 0;
 }
 
 static int vp56_size_changed(VP56Context *s)
@@ -653,7 +657,9 @@ static int ff_vp56_decode_mbs(AVCodecContext *avctx, void *data,
         s->block_offset[5] = s->block_offset[4];
 
         for (mb_col=0; mb_col<s->mb_width; mb_col++) {
-            vp56_decode_mb(s, mb_row, mb_col, is_alpha);
+            int ret = vp56_decode_mb(s, mb_row, mb_col, is_alpha);
+            if (ret < 0)
+                return ret;
 
             for (y=0; y<4; y++) {
                 s->above_block_idx[y] += 2;
diff --git a/libavcodec/vp56.h b/libavcodec/vp56.h
index 56c30919b7..34d48228fd 100644
--- a/libavcodec/vp56.h
+++ b/libavcodec/vp56.h
@@ -74,7 +74,7 @@ typedef void (*VP56ParseVectorAdjustment)(VP56Context *s,
 typedef void (*VP56Filter)(VP56Context *s, uint8_t *dst, uint8_t *src,
                            int offset1, int offset2, int stride,
                            VP56mv mv, int mask, int select, int luma);
-typedef void (*VP56ParseCoeff)(VP56Context *s);
+typedef int  (*VP56ParseCoeff)(VP56Context *s);
 typedef void (*VP56DefaultModelsInit)(VP56Context *s);
 typedef void (*VP56ParseVectorModels)(VP56Context *s);
 typedef int  (*VP56ParseCoeffModels)(VP56Context *s);
diff --git a/libavcodec/vp6.c b/libavcodec/vp6.c
index a2bb4578d5..7f0a9b7d5d 100644
--- a/libavcodec/vp6.c
+++ b/libavcodec/vp6.c
@@ -40,8 +40,8 @@
 
 #define VP6_MAX_HUFF_SIZE 12
 
-static void vp6_parse_coeff(VP56Context *s);
-static void vp6_parse_coeff_huffman(VP56Context *s);
+static int vp6_parse_coeff(VP56Context *s);
+static int vp6_parse_coeff_huffman(VP56Context *s);
 
 static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size)
 {
@@ -380,7 +380,7 @@ static unsigned vp6_get_nb_null(VP56Context *s)
     return val;
 }
 
-static void vp6_parse_coeff_huffman(VP56Context *s)
+static int vp6_parse_coeff_huffman(VP56Context *s)
 {
     VP56Model *model = s->modelp;
     uint8_t *permute = s->idct_scantable;
@@ -402,7 +402,7 @@ static void vp6_parse_coeff_huffman(VP56Context *s)
                     break;
             } else {
                 if (get_bits_left(&s->gb) <= 0)
-                    return;
+                    return AVERROR_INVALIDDATA;
                 coeff = get_vlc2(&s->gb, vlc_coeff->table, FF_HUFFMAN_BITS, 3);
                 if (coeff == 0) {
                     if (coeff_idx) {
@@ -437,9 +437,10 @@ static void vp6_parse_coeff_huffman(VP56Context *s)
             vlc_coeff = &s->ract_vlc[pt][ct][cg];
         }
     }
+    return 0;
 }
 
-static void vp6_parse_coeff(VP56Context *s)
+static int vp6_parse_coeff(VP56Context *s)
 {
     VP56RangeCoder *c = s->ccp;
     VP56Model *model = s->modelp;
@@ -449,6 +450,11 @@ static void vp6_parse_coeff(VP56Context *s)
     int b, i, cg, idx, ctx;
     int pt = 0;    /* plane type (0 for Y, 1 for U or V) */
 
+    if (c->end >= c->buffer && c->bits >= 0) {
+        av_log(s->avctx, AV_LOG_ERROR, "End of AC stream reached in vp6_parse_coeff\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     for (b=0; b<6; b++) {
         int ct = 1;    /* code type */
         int run = 1;
@@ -512,6 +518,7 @@ static void vp6_parse_coeff(VP56Context *s)
         s->left_block[ff_vp56_b6to4[b]].not_null_dc =
         s->above_blocks[s->above_block_idx[b]].not_null_dc = !!s->block_coeff[b][0];
     }
+    return 0;
 }
 
 static int vp6_block_variance(uint8_t *src, int stride)