Merge remote-tracking branch 'qatar/release/0.5' into release/0.5

* qatar/release/0.5:
  matroskadec: Fix a bug where a pointer was cached to an array that might later move due to a realloc()
  vorbis: Avoid some out-of-bounds reads
  vp3: fix oob read for negative tokens and memleaks on error.

Merged-by: Michael Niedermayer <michaelni@gmx.at>

3 files changed, 29 insertions, 6 deletions

diff --git a/libavcodec/vorbis.c b/libavcodec/vorbis.c
index dbc409f8d7..13e7e65b0f 100644
--- a/libavcodec/vorbis.c
+++ b/libavcodec/vorbis.c
@@ -146,13 +146,13 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values) {
     }
 }
 
-static void render_line(int x0, int y0, int x1, int y1, float * buf) {
+static void render_line(int x0, uint8_t y0, int x1, int y1, float * buf) {
     int dy = y1 - y0;
     int adx = x1 - x0;
     int base = dy / adx;
     int ady = FFABS(dy) - FFABS(base) * adx;
     int x = x0;
-    int y = y0;
+    uint8_t y = y0;
     int err = 0;
     int sy = dy<0 ? -1 : 1;
     buf[x] = ff_vorbis_floor1_inverse_db_table[y];
@@ -168,7 +168,8 @@ static void render_line(int x0, int y0, int x1, int y1, float * buf) {
 }
 
 void ff_vorbis_floor1_render_list(vorbis_floor1_entry * list, int values, uint_fast16_t * y_list, int * flag, int multiplier, float * out, int samples) {
-    int lx, ly, i;
+    int lx, i;
+    uint8_t ly;
     lx = 0;
     ly = y_list[0] * multiplier;
     for (i = 1; i < values; i++) {
diff --git a/libavcodec/vp3.c b/libavcodec/vp3.c
index 429c4f98a4..69248d6775 100644
--- a/libavcodec/vp3.c
+++ b/libavcodec/vp3.c
@@ -1011,12 +1011,12 @@ static int unpack_vlcs(Vp3DecodeContext *s, GetBitContext *gb,
             /* decode a VLC into a token */
             token = get_vlc2(gb, table->table, 5, 3);
             /* use the token to get a zero run, a coefficient, and an eob run */
-            if (token <= 6) {
+            if ((unsigned) token <= 6U) {
                 eob_run = eob_run_base[token];
                 if (eob_run_get_bits[token])
                     eob_run += get_bits(gb, eob_run_get_bits[token]);
                 coeff = zero_run = 0;
-            } else {
+            } else if (token >= 0) {
                 bits_to_get = coeff_get_bits[token];
                 if (!bits_to_get)
                     coeff = coeff_tables[token][0];
@@ -1026,6 +1026,10 @@ static int unpack_vlcs(Vp3DecodeContext *s, GetBitContext *gb,
                 zero_run = zero_run_base[token];
                 if (zero_run_get_bits[token])
                     zero_run += get_bits(gb, zero_run_get_bits[token]);
+            } else {
+                av_log(s->avctx, AV_LOG_ERROR,
+                       "Invalid token %d\n", token);
+                return -1;
             }
         }
 
@@ -1071,6 +1075,8 @@ static int unpack_dct_coeffs(Vp3DecodeContext *s, GetBitContext *gb)
     /* unpack the C plane DC coefficients */
     residual_eob_run = unpack_vlcs(s, gb, &s->dc_vlc[dc_c_table], 0,
         s->first_coded_c_fragment, s->last_coded_c_fragment, residual_eob_run);
+    if (residual_eob_run < 0)
+        return residual_eob_run;
 
     /* fetch the AC table indexes */
     ac_y_table = get_bits(gb, 4);
@@ -1080,36 +1086,52 @@ static int unpack_dct_coeffs(Vp3DecodeContext *s, GetBitContext *gb)
     for (i = 1; i <= 5; i++) {
         residual_eob_run = unpack_vlcs(s, gb, &s->ac_vlc_1[ac_y_table], i,
             s->first_coded_y_fragment, s->last_coded_y_fragment, residual_eob_run);
+        if (residual_eob_run < 0)
+            return residual_eob_run;
 
         residual_eob_run = unpack_vlcs(s, gb, &s->ac_vlc_1[ac_c_table], i,
             s->first_coded_c_fragment, s->last_coded_c_fragment, residual_eob_run);
+        if (residual_eob_run < 0)
+            return residual_eob_run;
     }
 
     /* unpack the group 2 AC coefficients (coeffs 6-14) */
     for (i = 6; i <= 14; i++) {
         residual_eob_run = unpack_vlcs(s, gb, &s->ac_vlc_2[ac_y_table], i,
             s->first_coded_y_fragment, s->last_coded_y_fragment, residual_eob_run);
+        if (residual_eob_run < 0)
+            return residual_eob_run;
 
         residual_eob_run = unpack_vlcs(s, gb, &s->ac_vlc_2[ac_c_table], i,
             s->first_coded_c_fragment, s->last_coded_c_fragment, residual_eob_run);
+        if (residual_eob_run < 0)
+            return residual_eob_run;
     }
 
     /* unpack the group 3 AC coefficients (coeffs 15-27) */
     for (i = 15; i <= 27; i++) {
         residual_eob_run = unpack_vlcs(s, gb, &s->ac_vlc_3[ac_y_table], i,
             s->first_coded_y_fragment, s->last_coded_y_fragment, residual_eob_run);
+        if (residual_eob_run < 0)
+            return residual_eob_run;
 
         residual_eob_run = unpack_vlcs(s, gb, &s->ac_vlc_3[ac_c_table], i,
             s->first_coded_c_fragment, s->last_coded_c_fragment, residual_eob_run);
+        if (residual_eob_run < 0)
+            return residual_eob_run;
     }
 
     /* unpack the group 4 AC coefficients (coeffs 28-63) */
     for (i = 28; i <= 63; i++) {
         residual_eob_run = unpack_vlcs(s, gb, &s->ac_vlc_4[ac_y_table], i,
             s->first_coded_y_fragment, s->last_coded_y_fragment, residual_eob_run);
+        if (residual_eob_run < 0)
+            return residual_eob_run;
 
         residual_eob_run = unpack_vlcs(s, gb, &s->ac_vlc_4[ac_c_table], i,
             s->first_coded_c_fragment, s->last_coded_c_fragment, residual_eob_run);
+        if (residual_eob_run < 0)
+            return residual_eob_run;
     }
 
     return 0;
diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c
index 0d75c5c13f..7350562402 100644
--- a/libavformat/matroskadec.c
+++ b/libavformat/matroskadec.c
@@ -1063,13 +1063,13 @@ static void matroska_convert_tags(AVFormatContext *s)
 static void matroska_execute_seekhead(MatroskaDemuxContext *matroska)
 {
     EbmlList *seekhead_list = &matroska->seekhead;
-    MatroskaSeekhead *seekhead = seekhead_list->elem;
     uint32_t level_up = matroska->level_up;
     int64_t before_pos = url_ftell(matroska->ctx->pb);
     MatroskaLevel level;
     int i;
 
     for (i=0; i<seekhead_list->nb_elem; i++) {
+        MatroskaSeekhead *seekhead = seekhead_list->elem;
         int64_t offset = seekhead[i].pos + matroska->segment_start;
 
         if (seekhead[i].pos <= before_pos