Merge remote-tracking branch 'qatar/release/0.6' into release/0.6

* qatar/release/0.6: (58 commits)
  Bump version number for 0.6.4 release.
  qdm2: check output buffer size before decoding
  Fix qdm2 decoder packet handling to match the api
  4xm: Add a check in decode_i_frame to prevent buffer overreads
  wma: initialize prev_block_len_bits, next_block_len_bits, and block_len_bits.
  swscale: #include "libavutil/mathematics.h"
  vp3dec: Check coefficient index in vp3_dequant()
  svq1dec: call avcodec_set_dimensions() after dimensions changed.
  vp6: Fix illegal read.
  vp6: Fix illegal read.
  vp6: Reset the internal state when aborting key frames header parsing
  vp6: Check for huffman tree build errors
  vp6: partially propagate huffman tree building errors during coeff model parsing and fix misspelling
  Fix out of bound reads in the QDM2 decoder.
  Check for out of bound writes in the QDM2 decoder.
  vmd: fix segfaults on corruped streams
  rv34: Check for invalid slice offsets
  rv34: Fix potential overreads
  rv34: Avoid NULL dereference on corrupted bitstream
  rv10: Reject slices that does not have the same type as the first one
  ...

Merged-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 18 insertions, 12 deletions

diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c
index 1d85f68907..e2c1a8776c 100644
--- a/libavcodec/smacker.c
+++ b/libavcodec/smacker.c
@@ -133,11 +133,13 @@ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx
             return -1;
         }
         b1 = get_bits_count(gb);
-        i1 = get_vlc2(gb, ctx->v1->table, SMKTREE_BITS, 3);
+        i1 = ctx->v1->table ? get_vlc2(gb, ctx->v1->table, SMKTREE_BITS, 3) : 0;
         b1 = get_bits_count(gb) - b1;
         b2 = get_bits_count(gb);
-        i2 = get_vlc2(gb, ctx->v2->table, SMKTREE_BITS, 3);
+        i2 = ctx->v2->table ? get_vlc2(gb, ctx->v2->table, SMKTREE_BITS, 3) : 0;
         b2 = get_bits_count(gb) - b2;
+        if (i1 < 0 || i2 < 0)
+            return -1;
         val = ctx->recode1[i1] | (ctx->recode2[i2] << 8);
         if(val == ctx->escapes[0]) {
             ctx->last[0] = hc->current;
@@ -289,7 +291,8 @@ static int decode_header_trees(SmackVContext *smk) {
         smk->mmap_tbl[0] = 0;
         smk->mmap_last[0] = smk->mmap_last[1] = smk->mmap_last[2] = 1;
     } else {
-        smacker_decode_header_tree(smk, &gb, &smk->mmap_tbl, smk->mmap_last, mmap_size);
+        if (smacker_decode_header_tree(smk, &gb, &smk->mmap_tbl, smk->mmap_last, mmap_size))
+            return -1;
     }
     if(!get_bits1(&gb)) {
         av_log(smk->avctx, AV_LOG_INFO, "Skipping MCLR tree\n");
@@ -297,7 +300,8 @@ static int decode_header_trees(SmackVContext *smk) {
         smk->mclr_tbl[0] = 0;
         smk->mclr_last[0] = smk->mclr_last[1] = smk->mclr_last[2] = 1;
     } else {
-        smacker_decode_header_tree(smk, &gb, &smk->mclr_tbl, smk->mclr_last, mclr_size);
+        if (smacker_decode_header_tree(smk, &gb, &smk->mclr_tbl, smk->mclr_last, mclr_size))
+            return -1;
     }
     if(!get_bits1(&gb)) {
         av_log(smk->avctx, AV_LOG_INFO, "Skipping FULL tree\n");
@@ -305,7 +309,8 @@ static int decode_header_trees(SmackVContext *smk) {
         smk->full_tbl[0] = 0;
         smk->full_last[0] = smk->full_last[1] = smk->full_last[2] = 1;
     } else {
-        smacker_decode_header_tree(smk, &gb, &smk->full_tbl, smk->full_last, full_size);
+        if (smacker_decode_header_tree(smk, &gb, &smk->full_tbl, smk->full_last, full_size))
+            return -1;
     }
     if(!get_bits1(&gb)) {
         av_log(smk->avctx, AV_LOG_INFO, "Skipping TYPE tree\n");
@@ -313,7 +318,8 @@ static int decode_header_trees(SmackVContext *smk) {
         smk->type_tbl[0] = 0;
         smk->type_last[0] = smk->type_last[1] = smk->type_last[2] = 1;
     } else {
-        smacker_decode_header_tree(smk, &gb, &smk->type_tbl, smk->type_last, type_size);
+        if (smacker_decode_header_tree(smk, &gb, &smk->type_tbl, smk->type_last, type_size))
+            return -1;
     }
 
     return 0;
@@ -523,8 +529,8 @@ static av_cold int decode_init(AVCodecContext *avctx)
         return -1;
     }
 
-    decode_header_trees(c);
-
+    if (decode_header_trees(c))
+        return -1;
 
     return 0;
 }
@@ -619,9 +625,9 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
     if(bits) { //decode 16-bit data
         for(i = stereo; i >= 0; i--)
             pred[i] = bswap_16(get_bits(&gb, 16));
-        for(i = 0; i < stereo; i++)
+        for(i = 0; i <= stereo; i++)
             *samples++ = pred[i];
-        for(i = 0; i < unp_size / 2; i++) {
+        for(; i < unp_size / 2; i++) {
             if(i & stereo) {
                 if(vlc[2].table)
                     res = get_vlc2(&gb, vlc[2].table, SMKTREE_BITS, 3);
@@ -653,9 +659,9 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
     } else { //8-bit data
         for(i = stereo; i >= 0; i--)
             pred[i] = get_bits(&gb, 8);
-        for(i = 0; i < stereo; i++)
+        for(i = 0; i <= stereo; i++)
             *samples8++ = pred[i];
-        for(i = 0; i < unp_size; i++) {
+        for(; i < unp_size; i++) {
             if(i & stereo){
                 if(vlc[1].table)
                     res = get_vlc2(&gb, vlc[1].table, SMKTREE_BITS, 3);