Merge remote-tracking branch 'qatar/release/0.6' into release/0.6

* qatar/release/0.6: (58 commits)
  Bump version number for 0.6.4 release.
  qdm2: check output buffer size before decoding
  Fix qdm2 decoder packet handling to match the api
  4xm: Add a check in decode_i_frame to prevent buffer overreads
  wma: initialize prev_block_len_bits, next_block_len_bits, and block_len_bits.
  swscale: #include "libavutil/mathematics.h"
  vp3dec: Check coefficient index in vp3_dequant()
  svq1dec: call avcodec_set_dimensions() after dimensions changed.
  vp6: Fix illegal read.
  vp6: Fix illegal read.
  vp6: Reset the internal state when aborting key frames header parsing
  vp6: Check for huffman tree build errors
  vp6: partially propagate huffman tree building errors during coeff model parsing and fix misspelling
  Fix out of bound reads in the QDM2 decoder.
  Check for out of bound writes in the QDM2 decoder.
  vmd: fix segfaults on corruped streams
  rv34: Check for invalid slice offsets
  rv34: Fix potential overreads
  rv34: Avoid NULL dereference on corrupted bitstream
  rv10: Reject slices that does not have the same type as the first one
  ...

Merged-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 9 insertions, 7 deletions

diff --git a/libavcodec/rv34.c b/libavcodec/rv34.c
index 88652f9fe1..e6fa3cfd8f 100644
--- a/libavcodec/rv34.c
+++ b/libavcodec/rv34.c
@@ -1439,12 +1439,14 @@ int ff_rv34_decode_frame(AVCodecContext *avctx,
         slice_count = (*buf++) + 1;
         slices_hdr = buf + 4;
         buf += 8 * slice_count;
+        buf_size -= 1 + 8 * slice_count;
     }else
         slice_count = avctx->slice_count;
 
     //parse first slice header to check whether this frame can be decoded
-    if(get_slice_offset(avctx, slices_hdr, 0) > buf_size){
-        av_log(avctx, AV_LOG_ERROR, "Slice offset is greater than frame size\n");
+    if(get_slice_offset(avctx, slices_hdr, 0) < 0 ||
+       get_slice_offset(avctx, slices_hdr, 0) > buf_size){
+        av_log(avctx, AV_LOG_ERROR, "Slice offset is invalid\n");
         return -1;
     }
     init_get_bits(&s->gb, buf+get_slice_offset(avctx, slices_hdr, 0), buf_size-get_slice_offset(avctx, slices_hdr, 0));
@@ -1459,7 +1461,7 @@ int ff_rv34_decode_frame(AVCodecContext *avctx,
     if(   (avctx->skip_frame >= AVDISCARD_NONREF && si.type==FF_B_TYPE)
        || (avctx->skip_frame >= AVDISCARD_NONKEY && si.type!=FF_I_TYPE)
        ||  avctx->skip_frame >= AVDISCARD_ALL)
-        return buf_size;
+        return avpkt->size;
     /* skip everything if we are in a hurry>=5 */
     if(avctx->hurry_up>=5)
         return buf_size;
@@ -1472,8 +1474,8 @@ int ff_rv34_decode_frame(AVCodecContext *avctx,
         else
             size= get_slice_offset(avctx, slices_hdr, i+1) - offset;
 
-        if(offset > buf_size){
-            av_log(avctx, AV_LOG_ERROR, "Slice offset is greater than frame size\n");
+        if(offset < 0 || offset > buf_size || size < 0){
+            av_log(avctx, AV_LOG_ERROR, "Slice offset is invalid\n");
             break;
         }
 
@@ -1494,7 +1496,7 @@ int ff_rv34_decode_frame(AVCodecContext *avctx,
             break;
     }
 
-    if(last){
+    if(last && s->current_picture_ptr){
         if(r->loop_filter)
             r->loop_filter(r, s->mb_height - 1);
         ff_er_frame_end(s);
@@ -1511,7 +1513,7 @@ int ff_rv34_decode_frame(AVCodecContext *avctx,
         }
         s->current_picture_ptr= NULL; //so we can detect if frame_end wasnt called (find some nicer solution...)
     }
-    return buf_size;
+    return avpkt->size;
 }
 
 av_cold int ff_rv34_decode_end(AVCodecContext *avctx)