Merge remote-tracking branch 'qatar/release/0.6' into release/0.6

* qatar/release/0.6: (58 commits)
  Bump version number for 0.6.4 release.
  qdm2: check output buffer size before decoding
  Fix qdm2 decoder packet handling to match the api
  4xm: Add a check in decode_i_frame to prevent buffer overreads
  wma: initialize prev_block_len_bits, next_block_len_bits, and block_len_bits.
  swscale: #include "libavutil/mathematics.h"
  vp3dec: Check coefficient index in vp3_dequant()
  svq1dec: call avcodec_set_dimensions() after dimensions changed.
  vp6: Fix illegal read.
  vp6: Fix illegal read.
  vp6: Reset the internal state when aborting key frames header parsing
  vp6: Check for huffman tree build errors
  vp6: partially propagate huffman tree building errors during coeff model parsing and fix misspelling
  Fix out of bound reads in the QDM2 decoder.
  Check for out of bound writes in the QDM2 decoder.
  vmd: fix segfaults on corruped streams
  rv34: Check for invalid slice offsets
  rv34: Fix potential overreads
  rv34: Avoid NULL dereference on corrupted bitstream
  rv10: Reject slices that does not have the same type as the first one
  ...

Merged-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 27 insertions, 10 deletions

diff --git a/libavcodec/qdm2.c b/libavcodec/qdm2.c
index 6451fbe91f..839e3fe1bf 100644
--- a/libavcodec/qdm2.c
+++ b/libavcodec/qdm2.c
@@ -75,6 +75,7 @@ do { \
 #define SAMPLES_NEEDED_2(why) \
      av_log (NULL,AV_LOG_INFO,"This file triggers some missing code. Please contact the developers.\nPosition: %s\n",why);
 
+#define QDM2_MAX_FRAME_SIZE 512
 
 typedef int8_t sb_int8_array[2][30][64];
 
@@ -167,7 +168,7 @@ typedef struct {
     /// I/O data
     const uint8_t *compressed_data;
     int compressed_size;
-    float output_buffer[1024];
+    float output_buffer[QDM2_MAX_FRAME_SIZE * 2];
 
     /// Synthesis filter
     DECLARE_ALIGNED(16, MPA_INT, synth_buf)[MPA_MAX_CHANNELS][512*2];
@@ -1355,6 +1356,8 @@ static void qdm2_fft_decode_tones (QDM2Context *q, int duration, GetBitContext *
             return;
 
         local_int_14 = (offset >> local_int_8);
+        if (local_int_14 >= FF_ARRAY_ELEMS(fft_level_index_table))
+            return;
 
         if (q->nb_channels > 1) {
             channel = get_bits1(gb);
@@ -1799,6 +1802,8 @@ static av_cold int qdm2_decode_init(AVCodecContext *avctx)
 
     avctx->channels = s->nb_channels = s->channels = AV_RB32(extradata);
     extradata += 4;
+    if (s->channels > MPA_MAX_CHANNELS)
+        return AVERROR_INVALIDDATA;
 
     avctx->sample_rate = AV_RB32(extradata);
     extradata += 4;
@@ -1820,6 +1825,8 @@ static av_cold int qdm2_decode_init(AVCodecContext *avctx)
     // something like max decodable tones
     s->group_order = av_log2(s->group_size) + 1;
     s->frame_size = s->group_size / 16; // 16 iterations per super block
+    if (s->frame_size > QDM2_MAX_FRAME_SIZE)
+        return AVERROR_INVALIDDATA;
 
     s->sub_sampling = s->fft_order - 7;
     s->frequency_range = 255 / (1 << (2 - s->sub_sampling));
@@ -1883,7 +1890,7 @@ static av_cold int qdm2_decode_close(AVCodecContext *avctx)
 }
 
 
-static void qdm2_decode (QDM2Context *q, const uint8_t *in, int16_t *out)
+static int qdm2_decode (QDM2Context *q, const uint8_t *in, int16_t *out)
 {
     int ch, i;
     const int frame_size = (q->frame_size * q->channels);
@@ -1919,7 +1926,7 @@ static void qdm2_decode (QDM2Context *q, const uint8_t *in, int16_t *out)
 
         if (!q->has_errors && q->sub_packet_list_C[0].packet != NULL) {
             SAMPLES_NEEDED_2("has errors, and C list is not empty")
-            return;
+            return -1;
         }
     }
 
@@ -1940,6 +1947,8 @@ static void qdm2_decode (QDM2Context *q, const uint8_t *in, int16_t *out)
 
         out[i] = value;
     }
+
+    return 0;
 }
 
 
@@ -1950,25 +1959,33 @@ static int qdm2_decode_frame(AVCodecContext *avctx,
     const uint8_t *buf = avpkt->data;
     int buf_size = avpkt->size;
     QDM2Context *s = avctx->priv_data;
+    int16_t *out = data;
+    int i, out_size;
 
     if(!buf)
         return 0;
     if(buf_size < s->checksum_size)
         return -1;
 
-    *data_size = s->channels * s->frame_size * sizeof(int16_t);
+    out_size = 16 * s->channels * s->frame_size *
+               av_get_bits_per_sample_format(avctx->sample_fmt)/8;
+    if (*data_size < out_size) {
+        av_log(avctx, AV_LOG_ERROR, "Output buffer is too small\n");
+        return AVERROR(EINVAL);
+    }
 
     av_log(avctx, AV_LOG_DEBUG, "decode(%d): %p[%d] -> %p[%d]\n",
        buf_size, buf, s->checksum_size, data, *data_size);
 
-    qdm2_decode(s, buf, data);
-
-    // reading only when next superblock found
-    if (s->sub_packet == 0) {
-        return s->checksum_size;
+    for (i = 0; i < 16; i++) {
+        if (qdm2_decode(s, buf, out) < 0)
+            return -1;
+        out += s->channels * s->frame_size;
     }
 
-    return 0;
+    *data_size = out_size;
+
+    return buf_size;
 }
 
 AVCodec qdm2_decoder =