huffyuv: Check and propagate function return values

Bug-Id: CVE-2013-0868

inspired by a patch from Michael Niedermayer <michaelni@gmx.at>
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Diego Biurrun <diego@biurrun.de>

CC: libav-stable@libav.org
(cherry picked from commit d0393d79bc3d61c9f2ff832c0e273b7774ff0269)
Signed-off-by: Diego Biurrun <diego@biurrun.de>

Conflicts:
	libavcodec/huffyuvdec.c

1 files changed, 61 insertions, 45 deletions

diff --git a/libavcodec/huffyuvdec.c b/libavcodec/huffyuvdec.c
index ed490d4bd7..016b0d6751 100644
--- a/libavcodec/huffyuvdec.c
+++ b/libavcodec/huffyuvdec.c
@@ -104,11 +104,13 @@ static int read_len_table(uint8_t *dst, GetBitContext *gb)
     return 0;
 }
 
-static void generate_joint_tables(HYuvContext *s)
+static int generate_joint_tables(HYuvContext *s)
 {
     uint16_t symbols[1 << VLC_BITS];
     uint16_t bits[1 << VLC_BITS];
     uint8_t len[1 << VLC_BITS];
+    int ret;
+
     if (s->bitstream_bpp < 24) {
         int p, i, y, u;
         for (p = 0; p < 3; p++) {
@@ -129,8 +131,9 @@ static void generate_joint_tables(HYuvContext *s)
                 }
             }
             ff_free_vlc(&s->vlc[3 + p]);
-            ff_init_vlc_sparse(&s->vlc[3 + p], VLC_BITS, i, len, 1, 1,
-                               bits, 2, 2, symbols, 2, 2, 0);
+            if ((ret = ff_init_vlc_sparse(&s->vlc[3 + p], VLC_BITS, i, len, 1, 1,
+                                          bits, 2, 2, symbols, 2, 2, 0)) < 0)
+                return ret;
         }
     } else {
         uint8_t (*map)[4] = (uint8_t(*)[4])s->pix_bgr_map;
@@ -171,29 +174,34 @@ static void generate_joint_tables(HYuvContext *s)
             }
         }
         ff_free_vlc(&s->vlc[3]);
-        init_vlc(&s->vlc[3], VLC_BITS, i, len, 1, 1, bits, 2, 2, 0);
+        if ((ret = init_vlc(&s->vlc[3], VLC_BITS, i, len, 1, 1,
+                            bits, 2, 2, 0)) < 0)
+            return ret;
     }
+    return 0;
 }
 
 static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length)
 {
     GetBitContext gb;
-    int i;
+    int i, ret;
 
-    init_get_bits(&gb, src, length * 8);
+    if ((ret = init_get_bits(&gb, src, length * 8)) < 0)
+        return ret;
 
     for (i = 0; i < 3; i++) {
-        if (read_len_table(s->len[i], &gb) < 0)
-            return -1;
-        if (ff_huffyuv_generate_bits_table(s->bits[i], s->len[i]) < 0) {
-            return -1;
-        }
+        if ((ret = read_len_table(s->len[i], &gb)) < 0)
+            return ret;
+        if ((ret = ff_huffyuv_generate_bits_table(s->bits[i], s->len[i])) < 0)
+            return ret;
         ff_free_vlc(&s->vlc[i]);
-        init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1,
-                 s->bits[i], 4, 4, 0);
+        if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1,
+                            s->bits[i], 4, 4, 0)) < 0)
+            return ret;
     }
 
-    generate_joint_tables(s);
+    if ((ret = generate_joint_tables(s)) < 0)
+        return ret;
 
     return (get_bits_count(&gb) + 7) / 8;
 }
@@ -201,17 +209,19 @@ static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length)
 static int read_old_huffman_tables(HYuvContext *s)
 {
     GetBitContext gb;
-    int i;
+    int i, ret;
 
-    init_get_bits(&gb, classic_shift_luma,
-                  classic_shift_luma_table_size * 8);
-    if (read_len_table(s->len[0], &gb) < 0)
-        return -1;
+    if ((ret = init_get_bits(&gb, classic_shift_luma,
+                             classic_shift_luma_table_size * 8)) < 0)
+        return ret;
+    if ((ret = read_len_table(s->len[0], &gb)) < 0)
+        return ret;
 
-    init_get_bits(&gb, classic_shift_chroma,
-                  classic_shift_chroma_table_size * 8);
-    if (read_len_table(s->len[1], &gb) < 0)
-        return -1;
+    if ((ret = init_get_bits(&gb, classic_shift_chroma,
+                             classic_shift_chroma_table_size * 8)) < 0)
+        return ret;
+    if ((ret = read_len_table(s->len[1], &gb)) < 0)
+        return ret;
 
     for(i=0; i<256; i++) s->bits[0][i] = classic_add_luma  [i];
     for(i=0; i<256; i++) s->bits[1][i] = classic_add_chroma[i];
@@ -225,11 +235,13 @@ static int read_old_huffman_tables(HYuvContext *s)
 
     for (i = 0; i < 3; i++) {
         ff_free_vlc(&s->vlc[i]);
-        init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1,
-                 s->bits[i], 4, 4, 0);
+        if ((ret = init_vlc(&s->vlc[i], VLC_BITS, 256, s->len[i], 1, 1,
+                            s->bits[i], 4, 4, 0)) < 0)
+            return ret;
     }
 
-    generate_joint_tables(s);
+    if ((ret = generate_joint_tables(s)) < 0)
+        return ret;
 
     return 0;
 }
@@ -237,6 +249,7 @@ static int read_old_huffman_tables(HYuvContext *s)
 static av_cold int decode_init(AVCodecContext *avctx)
 {
     HYuvContext *s = avctx->priv_data;
+    int ret;
 
     ff_huffyuv_common_init(avctx);
     memset(s->vlc, 0, 3 * sizeof(VLC));
@@ -270,9 +283,9 @@ static av_cold int decode_init(AVCodecContext *avctx)
         s->interlaced = (interlace == 1) ? 1 : (interlace == 2) ? 0 : s->interlaced;
         s->context = ((uint8_t*)avctx->extradata)[2] & 0x40 ? 1 : 0;
 
-        if ( read_huffman_tables(s, ((uint8_t*)avctx->extradata) + 4,
-                                 avctx->extradata_size - 4) < 0)
-            return -1;
+        if ((ret = read_huffman_tables(s, ((uint8_t*)avctx->extradata) + 4,
+                                       avctx->extradata_size - 4)) < 0)
+            return ret;
     }else{
         switch (avctx->bits_per_coded_sample & 7) {
         case 1:
@@ -299,8 +312,8 @@ static av_cold int decode_init(AVCodecContext *avctx)
         s->bitstream_bpp = avctx->bits_per_coded_sample & ~7;
         s->context = 0;
 
-        if (read_old_huffman_tables(s) < 0)
-            return -1;
+        if ((ret = read_old_huffman_tables(s)) < 0)
+            return ret;
     }
 
     switch (s->bitstream_bpp) {
@@ -326,7 +339,8 @@ static av_cold int decode_init(AVCodecContext *avctx)
         return AVERROR_INVALIDDATA;
     }
 
-    ff_huffyuv_alloc_temp(s);
+    if ((ret = ff_huffyuv_alloc_temp(s)) < 0)
+        return ret;
 
     return 0;
 }
@@ -334,20 +348,21 @@ static av_cold int decode_init(AVCodecContext *avctx)
 static av_cold int decode_init_thread_copy(AVCodecContext *avctx)
 {
     HYuvContext *s = avctx->priv_data;
-    int i;
+    int i, ret;
 
-    ff_huffyuv_alloc_temp(s);
+    if ((ret = ff_huffyuv_alloc_temp(s)) < 0)
+        return ret;
 
     for (i = 0; i < 6; i++)
         s->vlc[i].table = NULL;
 
     if (s->version == 2) {
-        if (read_huffman_tables(s, ((uint8_t*)avctx->extradata) + 4,
-                                avctx->extradata_size) < 0)
-            return -1;
+        if ((ret = read_huffman_tables(s, ((uint8_t*)avctx->extradata) + 4,
+                                       avctx->extradata_size)) < 0)
+            return ret;
     } else {
-        if (read_old_huffman_tables(s) < 0)
-            return -1;
+        if ((ret = read_old_huffman_tables(s)) < 0)
+            return ret;
     }
 
     return 0;
@@ -482,7 +497,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     int fake_ystride, fake_ustride, fake_vstride;
     ThreadFrame frame = { .f = data };
     AVFrame * const p = data;
-    int table_size = 0;
+    int table_size = 0, ret;
 
     av_fast_malloc(&s->bitstream_buffer,
                    &s->bitstream_buffer_size,
@@ -494,22 +509,23 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     s->dsp.bswap_buf((uint32_t*)s->bitstream_buffer,
                      (const uint32_t*)buf, buf_size / 4);
 
-    if (ff_thread_get_buffer(avctx, &frame, 0) < 0) {
+    if ((ret = ff_thread_get_buffer(avctx, &frame, 0)) < 0) {
         av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
-        return -1;
+        return ret;
     }
 
     if (s->context) {
         table_size = read_huffman_tables(s, s->bitstream_buffer, buf_size);
         if (table_size < 0)
-            return -1;
+            return table_size;
     }
 
     if ((unsigned)(buf_size-table_size) >= INT_MAX / 8)
         return -1;
 
-    init_get_bits(&s->gb, s->bitstream_buffer+table_size,
-                  (buf_size-table_size) * 8);
+    if ((ret = init_get_bits(&s->gb, s->bitstream_buffer + table_size,
+                             (buf_size - table_size) * 8)) < 0)
+        return ret;
 
     fake_ystride = s->interlaced ? p->linesize[0] * 2  : p->linesize[0];
     fake_ustride = s->interlaced ? p->linesize[1] * 2  : p->linesize[1];