Merge branch 'release/0.8' into release/0.7

* release/0.8: (185 commits)
  h264: fix intra 16x16 mode check when using mbaff and constrained_intra_pred.
  h264: check for invalid bit depth value.
  h264: add entries for 11 and 12 bits in ff_h264_chroma_qp[][]
  h264: fix the check for invalid SPS:num_ref_frames.
  h264: do not let invalid values in h->ref_count on ff_h264_decode_ref_pic_list_reordering() errors.
  Reject video with non multiple of 16 width/height in the 4xm decoder.
  4xm decoder: fix data size for i2 frames.
  4xm decoder: print some error messages in case of errors.
  Check for out of bound accesses in the 4xm decoder.
  Prevent block size from inreasing in the shorten decoder.
  Check for out of bound reads in PTX decoder.
  Clear FF_INPUT_BUFFER_PADDING_SIZE bytes at the end of the temporary buffers used in 4xm decoder.
  Fix the check for missing references in ff_er_frame_end() for H264.
  Prevent NULL dereference when the huffman table is invalid in the 4xm decoder.
  Fix use of uninitialized memory in 4X Technologies demuxer.
  h264: increase ref_poc size to 32 as it can be per field.
  h264: set unused ref_counts to 0 as a precautionary meassure.
  Remove Chnagelog it has nothing to do with reality
  fate: fix motion pixels checksum change caused by backported bugfix
  avienc: Add a limit on the number of skiped frames muxed in a row.
  ...

Conflicts:
	Doxyfile
	RELEASE
	VERSION
	libavformat/utils.c

Merged-by: Michael Niedermayer <[email protected]>

1 files changed, 77 insertions, 19 deletions

diff --git a/libavcodec/4xm.c b/libavcodec/4xm.c
index d89b494b09..7344f4cd1f 100644
--- a/libavcodec/4xm.c
+++ b/libavcodec/4xm.c
@@ -133,7 +133,9 @@ typedef struct FourXContext{
     GetBitContext pre_gb;          ///< ac/dc prefix
     GetBitContext gb;
     const uint8_t *bytestream;
+    const uint8_t *bytestream_end;
     const uint16_t *wordstream;
+    const uint16_t *wordstream_end;
     int mv[256];
     VLC pre_vlc;
     int last_dc;
@@ -328,6 +330,10 @@ static void decode_p_block(FourXContext *f, uint16_t *dst, uint16_t *src, int lo
     assert(code>=0 && code<=6);
 
     if(code == 0){
+        if (f->bytestream_end - f->bytestream < 1){
+            av_log(f->avctx, AV_LOG_ERROR, "bytestream overread\n");
+            return;
+        }
         src += f->mv[ *f->bytestream++ ];
         if(start > src || src > end){
             av_log(f->avctx, AV_LOG_ERROR, "mv out of pic\n");
@@ -345,15 +351,31 @@ static void decode_p_block(FourXContext *f, uint16_t *dst, uint16_t *src, int lo
     }else if(code == 3 && f->version<2){
         mcdc(dst, src, log2w, h, stride, 1, 0);
     }else if(code == 4){
+        if (f->bytestream_end - f->bytestream < 1){
+            av_log(f->avctx, AV_LOG_ERROR, "bytestream overread\n");
+            return;
+        }
         src += f->mv[ *f->bytestream++ ];
         if(start > src || src > end){
             av_log(f->avctx, AV_LOG_ERROR, "mv out of pic\n");
             return;
         }
+        if (f->wordstream_end - f->wordstream < 1){
+            av_log(f->avctx, AV_LOG_ERROR, "wordstream overread\n");
+            return;
+        }
         mcdc(dst, src, log2w, h, stride, 1, av_le2ne16(*f->wordstream++));
     }else if(code == 5){
+        if (f->wordstream_end - f->wordstream < 1){
+            av_log(f->avctx, AV_LOG_ERROR, "wordstream overread\n");
+            return;
+        }
         mcdc(dst, src, log2w, h, stride, 0, av_le2ne16(*f->wordstream++));
     }else if(code == 6){
+        if (f->wordstream_end - f->wordstream < 2){
+            av_log(f->avctx, AV_LOG_ERROR, "wordstream overread\n");
+            return;
+        }
         if(log2w){
             dst[0] = av_le2ne16(*f->wordstream++);
             dst[1] = av_le2ne16(*f->wordstream++);
@@ -375,6 +397,8 @@ static int decode_p_frame(FourXContext *f, const uint8_t *buf, int length){
 
     if(f->version>1){
         extra=20;
+        if (length < extra)
+            return -1;
         bitstream_size= AV_RL32(buf+8);
         wordstream_size= AV_RL32(buf+12);
         bytestream_size= AV_RL32(buf+16);
@@ -385,11 +409,10 @@ static int decode_p_frame(FourXContext *f, const uint8_t *buf, int length){
         bytestream_size= FFMAX(length - bitstream_size - wordstream_size, 0);
     }
 
-    if(bitstream_size+ bytestream_size+ wordstream_size + extra != length
-       || bitstream_size  > (1<<26)
-       || bytestream_size > (1<<26)
-       || wordstream_size > (1<<26)
-       ){
+    if (bitstream_size > length ||
+        bytestream_size > length - bitstream_size ||
+        wordstream_size > length - bytestream_size - bitstream_size ||
+        extra > length - bytestream_size - bitstream_size - wordstream_size){
         av_log(f->avctx, AV_LOG_ERROR, "lengths %d %d %d %d\n", bitstream_size, bytestream_size, wordstream_size,
         bitstream_size+ bytestream_size+ wordstream_size - length);
         return -1;
@@ -399,10 +422,13 @@ static int decode_p_frame(FourXContext *f, const uint8_t *buf, int length){
     if (!f->bitstream_buffer)
         return AVERROR(ENOMEM);
     f->dsp.bswap_buf(f->bitstream_buffer, (const uint32_t*)(buf + extra), bitstream_size/4);
+    memset((uint8_t*)f->bitstream_buffer + bitstream_size, 0, FF_INPUT_BUFFER_PADDING_SIZE);
     init_get_bits(&f->gb, f->bitstream_buffer, 8*bitstream_size);
 
     f->wordstream= (const uint16_t*)(buf + extra + bitstream_size);
+    f->wordstream_end= f->wordstream + wordstream_size/2;
     f->bytestream= buf + extra + bitstream_size + wordstream_size;
+    f->bytestream_end = f->bytestream + bytestream_size;
 
     init_mv(f);
 
@@ -531,7 +557,7 @@ static int decode_i_mb(FourXContext *f){
     return 0;
 }
 
-static const uint8_t *read_huffman_tables(FourXContext *f, const uint8_t * const buf){
+static const uint8_t *read_huffman_tables(FourXContext *f, const uint8_t * const buf, int buf_size){
     int frequency[512];
     uint8_t flag[512];
     int up[512];
@@ -539,6 +565,7 @@ static const uint8_t *read_huffman_tables(FourXContext *f, const uint8_t * const
     int bits_tab[257];
     int start, end;
     const uint8_t *ptr= buf;
+    const uint8_t *ptr_end = buf + buf_size;
     int j;
 
     memset(frequency, 0, sizeof(frequency));
@@ -549,6 +576,8 @@ static const uint8_t *read_huffman_tables(FourXContext *f, const uint8_t * const
     for(;;){
         int i;
 
+        if (start <= end && ptr_end - ptr < end - start + 1 + 1)
+            return NULL;
         for(i=start; i<=end; i++){
             frequency[i]= *ptr++;
         }
@@ -601,9 +630,10 @@ static const uint8_t *read_huffman_tables(FourXContext *f, const uint8_t * const
         len_tab[j]= len;
     }
 
-    init_vlc(&f->pre_vlc, ACDC_VLC_BITS, 257,
-             len_tab , 1, 1,
-             bits_tab, 4, 4, 0);
+    if (init_vlc(&f->pre_vlc, ACDC_VLC_BITS, 257,
+                 len_tab , 1, 1,
+                 bits_tab, 4, 4, 0))
+        return NULL;
 
     return ptr;
 }
@@ -621,10 +651,13 @@ static int decode_i2_frame(FourXContext *f, const uint8_t *buf, int length){
     const int height= f->avctx->height;
     uint16_t *dst= (uint16_t*)f->current_picture.data[0];
     const int stride= f->current_picture.linesize[0]>>1;
+    const uint8_t *buf_end = buf + length;
 
     for(y=0; y<height; y+=16){
         for(x=0; x<width; x+=16){
             unsigned int color[4], bits;
+            if (buf_end - buf < 8)
+                return -1;
             memset(color, 0, sizeof(color));
 //warning following is purely guessed ...
             color[0]= bytestream_get_le16(&buf);
@@ -658,18 +691,23 @@ static int decode_i_frame(FourXContext *f, const uint8_t *buf, int length){
     uint16_t *dst= (uint16_t*)f->current_picture.data[0];
     const int stride= f->current_picture.linesize[0]>>1;
     const unsigned int bitstream_size= AV_RL32(buf);
-    const int token_count av_unused = AV_RL32(buf + bitstream_size + 8);
-    unsigned int prestream_size= 4*AV_RL32(buf + bitstream_size + 4);
-    const uint8_t *prestream= buf + bitstream_size + 12;
+    unsigned int prestream_size;
+    const uint8_t *prestream;
+
+    if (bitstream_size > (1<<26) || length < bitstream_size + 12)
+        return -1;
+    prestream_size = 4*AV_RL32(buf + bitstream_size + 4);
+    prestream = buf + bitstream_size + 12;
 
-    if(prestream_size + bitstream_size + 12 != length
-       || bitstream_size > (1<<26)
-       || prestream_size > (1<<26)){
+    if (prestream_size > (1<<26) ||
+        prestream_size != length - (bitstream_size + 12)){
         av_log(f->avctx, AV_LOG_ERROR, "size mismatch %d %d %d\n", prestream_size, bitstream_size, length);
         return -1;
     }
 
-    prestream= read_huffman_tables(f, prestream);
+    prestream= read_huffman_tables(f, prestream, buf + length - prestream);
+    if (!prestream)
+        return -1;
 
     init_get_bits(&f->gb, buf + 4, 8*bitstream_size);
 
@@ -679,6 +717,7 @@ static int decode_i_frame(FourXContext *f, const uint8_t *buf, int length){
     if (!f->bitstream_buffer)
         return AVERROR(ENOMEM);
     f->dsp.bswap_buf(f->bitstream_buffer, (const uint32_t*)prestream, prestream_size/4);
+    memset((uint8_t*)f->bitstream_buffer + prestream_size, 0, FF_INPUT_BUFFER_PADDING_SIZE);
     init_get_bits(&f->pre_gb, f->bitstream_buffer, 8*prestream_size);
 
     f->last_dc= 0*128*8*8;
@@ -710,6 +749,8 @@ static int decode_frame(AVCodecContext *avctx,
     AVFrame *p, temp;
     int i, frame_4cc, frame_size;
 
+    if (buf_size < 12)
+        return AVERROR_INVALIDDATA;
     frame_4cc= AV_RL32(buf);
     if(buf_size != AV_RL32(buf+4)+8 || buf_size < 20){
         av_log(f->avctx, AV_LOG_ERROR, "size mismatch %d %d\n", buf_size, AV_RL32(buf+4));
@@ -722,6 +763,11 @@ static int decode_frame(AVCodecContext *avctx,
         const int whole_size= AV_RL32(buf+16);
         CFrameBuffer *cfrm;
 
+        if (data_size < 0 || whole_size < 0){
+            av_log(f->avctx, AV_LOG_ERROR, "sizes invalid\n");
+            return AVERROR_INVALIDDATA;
+        }
+
         for(i=0; i<CFRAME_BUFFER_COUNT; i++){
             if(f->cfrm[i].id && f->cfrm[i].id < avctx->frame_number)
                 av_log(f->avctx, AV_LOG_ERROR, "lost c frame %d\n", f->cfrm[i].id);
@@ -738,6 +784,8 @@ static int decode_frame(AVCodecContext *avctx,
         }
         cfrm= &f->cfrm[i];
 
+        if (data_size > UINT_MAX -  cfrm->size - FF_INPUT_BUFFER_PADDING_SIZE)
+            return AVERROR_INVALIDDATA;
         cfrm->data= av_fast_realloc(cfrm->data, &cfrm->allocated_size, cfrm->size + data_size + FF_INPUT_BUFFER_PADDING_SIZE);
         if(!cfrm->data){ //explicit check needed as memcpy below might not catch a NULL
             av_log(f->avctx, AV_LOG_ERROR, "realloc falure");
@@ -781,12 +829,16 @@ static int decode_frame(AVCodecContext *avctx,
 
     if(frame_4cc == AV_RL32("ifr2")){
         p->pict_type= AV_PICTURE_TYPE_I;
-        if(decode_i2_frame(f, buf-4, frame_size) < 0)
+        if(decode_i2_frame(f, buf-4, frame_size+4) < 0){
+            av_log(f->avctx, AV_LOG_ERROR, "decode i2 frame failed\n");
             return -1;
+        }
     }else if(frame_4cc == AV_RL32("ifrm")){
         p->pict_type= AV_PICTURE_TYPE_I;
-        if(decode_i_frame(f, buf, frame_size) < 0)
+        if(decode_i_frame(f, buf, frame_size) < 0){
+            av_log(f->avctx, AV_LOG_ERROR, "decode i frame failed\n");
             return -1;
+        }
     }else if(frame_4cc == AV_RL32("pfrm") || frame_4cc == AV_RL32("pfr2")){
         if(!f->last_picture.data[0]){
             f->last_picture.reference= 1;
@@ -797,8 +849,10 @@ static int decode_frame(AVCodecContext *avctx,
         }
 
         p->pict_type= AV_PICTURE_TYPE_P;
-        if(decode_p_frame(f, buf, frame_size) < 0)
+        if(decode_p_frame(f, buf, frame_size) < 0){
+            av_log(f->avctx, AV_LOG_ERROR, "decode p frame failed\n");
             return -1;
+        }
     }else if(frame_4cc == AV_RL32("snd_")){
         av_log(avctx, AV_LOG_ERROR, "ignoring snd_ chunk length:%d\n", buf_size);
     }else{
@@ -831,6 +885,10 @@ static av_cold int decode_init(AVCodecContext *avctx){
         av_log(avctx, AV_LOG_ERROR, "extradata wrong or missing\n");
         return 1;
     }
+    if((avctx->width % 16) || (avctx->height % 16)) {
+        av_log(avctx, AV_LOG_ERROR, "unsupported width/height\n");
+        return AVERROR_INVALIDDATA;
+    }
 
     avcodec_get_frame_defaults(&f->current_picture);
     avcodec_get_frame_defaults(&f->last_picture);