Merge branch 'release/0.8' into release/0.7

* release/0.8: (185 commits)
  h264: fix intra 16x16 mode check when using mbaff and constrained_intra_pred.
  h264: check for invalid bit depth value.
  h264: add entries for 11 and 12 bits in ff_h264_chroma_qp[][]
  h264: fix the check for invalid SPS:num_ref_frames.
  h264: do not let invalid values in h->ref_count on ff_h264_decode_ref_pic_list_reordering() errors.
  Reject video with non multiple of 16 width/height in the 4xm decoder.
  4xm decoder: fix data size for i2 frames.
  4xm decoder: print some error messages in case of errors.
  Check for out of bound accesses in the 4xm decoder.
  Prevent block size from inreasing in the shorten decoder.
  Check for out of bound reads in PTX decoder.
  Clear FF_INPUT_BUFFER_PADDING_SIZE bytes at the end of the temporary buffers used in 4xm decoder.
  Fix the check for missing references in ff_er_frame_end() for H264.
  Prevent NULL dereference when the huffman table is invalid in the 4xm decoder.
  Fix use of uninitialized memory in 4X Technologies demuxer.
  h264: increase ref_poc size to 32 as it can be per field.
  h264: set unused ref_counts to 0 as a precautionary meassure.
  Remove Chnagelog it has nothing to do with reality
  fate: fix motion pixels checksum change caused by backported bugfix
  avienc: Add a limit on the number of skiped frames muxed in a row.
  ...

Conflicts:
	Doxyfile
	RELEASE
	VERSION
	libavformat/utils.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 72 insertions, 24 deletions

diff --git a/libavcodec/vmdav.c b/libavcodec/vmdav.c
index b9acfe921c..1f9694ea29 100644
--- a/libavcodec/vmdav.c
+++ b/libavcodec/vmdav.c
@@ -72,9 +72,11 @@ typedef struct VmdVideoContext {
 #define QUEUE_SIZE 0x1000
 #define QUEUE_MASK 0x0FFF
 
-static void lz_unpack(const unsigned char *src, unsigned char *dest, int dest_len)
+static void lz_unpack(const unsigned char *src, int src_len,
+                      unsigned char *dest, int dest_len)
 {
     const unsigned char *s;
+    const unsigned char *s_end;
     unsigned char *d;
     unsigned char *d_end;
     unsigned char queue[QUEUE_SIZE];
@@ -87,8 +89,12 @@ static void lz_unpack(const unsigned char *src, unsigned char *dest, int dest_le
     unsigned int i, j;
 
     s = src;
+    s_end = src + src_len;
     d = dest;
     d_end = d + dest_len;
+
+    if (s_end - s < 8)
+        return;
     dataleft = AV_RL32(s);
     s += 4;
     memset(queue, 0x20, QUEUE_SIZE);
@@ -101,10 +107,10 @@ static void lz_unpack(const unsigned char *src, unsigned char *dest, int dest_le
         speclen = 100;  /* no speclen */
     }
 
-    while (dataleft > 0) {
+    while (s_end - s > 0 && dataleft > 0) {
         tag = *s++;
         if ((tag == 0xFF) && (dataleft > 8)) {
-            if (d + 8 > d_end)
+            if (d_end - d < 8 || s_end - s < 8)
                 return;
             for (i = 0; i < 8; i++) {
                 queue[qpos++] = *d++ = *s++;
@@ -116,18 +122,23 @@ static void lz_unpack(const unsigned char *src, unsigned char *dest, int dest_le
                 if (dataleft == 0)
                     break;
                 if (tag & 0x01) {
-                    if (d + 1 > d_end)
+                    if (d_end - d < 1 || s_end - s < 1)
                         return;
                     queue[qpos++] = *d++ = *s++;
                     qpos &= QUEUE_MASK;
                     dataleft--;
                 } else {
+                    if (s_end - s < 2)
+                        return;
                     chainofs = *s++;
                     chainofs |= ((*s & 0xF0) << 4);
                     chainlen = (*s++ & 0x0F) + 3;
-                    if (chainlen == speclen)
+                    if (chainlen == speclen) {
+                        if (s_end - s < 1)
+                            return;
                         chainlen = *s++ + 0xF + 3;
-                    if (d + chainlen > d_end)
+                    }
+                    if (d_end - d < chainlen)
                         return;
                     for (j = 0; j < chainlen; j++) {
                         *d = queue[chainofs++ & QUEUE_MASK];
@@ -142,32 +153,39 @@ static void lz_unpack(const unsigned char *src, unsigned char *dest, int dest_le
     }
 }
 
-static int rle_unpack(const unsigned char *src, unsigned char *dest,
-    int src_len, int dest_len)
+static int rle_unpack(const unsigned char *src, int src_len, int src_count,
+                      unsigned char *dest, int dest_len)
 {
     const unsigned char *ps;
+    const unsigned char *ps_end;
     unsigned char *pd;
     int i, l;
     unsigned char *dest_end = dest + dest_len;
 
     ps = src;
+    ps_end = src + src_len;
     pd = dest;
-    if (src_len & 1)
+    if (src_count & 1) {
+        if (ps_end - ps < 1)
+            return 0;
         *pd++ = *ps++;
+    }
 
-    src_len >>= 1;
+    src_count >>= 1;
     i = 0;
     do {
+        if (ps_end - ps < 1)
+            break;
         l = *ps++;
         if (l & 0x80) {
             l = (l & 0x7F) * 2;
-            if (pd + l > dest_end)
+            if (dest_end - pd < l || ps_end - ps < l)
                 return ps - src;
             memcpy(pd, ps, l);
             ps += l;
             pd += l;
         } else {
-            if (pd + i > dest_end)
+            if (dest_end - pd < i || ps_end - ps < 2)
                 return ps - src;
             for (i = 0; i < l; i++) {
                 *pd++ = ps[0];
@@ -176,7 +194,7 @@ static int rle_unpack(const unsigned char *src, unsigned char *dest,
             ps += 2;
         }
         i += l;
-    } while (i < src_len);
+    } while (i < src_count);
 
     return ps - src;
 }
@@ -189,8 +207,10 @@ static void vmd_decode(VmdVideoContext *s)
 
     /* point to the start of the encoded data */
     const unsigned char *p = s->buf + 16;
+    const unsigned char *p_end = s->buf + s->size;
 
     const unsigned char *pb;
+    const unsigned char *pb_end;
     unsigned char meth;
     unsigned char *dp;   /* pointer to current frame */
     unsigned char *pp;   /* pointer to previous frame */
@@ -204,6 +224,16 @@ static void vmd_decode(VmdVideoContext *s)
     frame_y = AV_RL16(&s->buf[8]);
     frame_width = AV_RL16(&s->buf[10]) - frame_x + 1;
     frame_height = AV_RL16(&s->buf[12]) - frame_y + 1;
+    if (frame_x < 0 || frame_width < 0 ||
+        frame_x >= s->avctx->width ||
+        frame_width > s->avctx->width ||
+        frame_x + frame_width > s->avctx->width)
+        return;
+    if (frame_y < 0 || frame_height < 0 ||
+        frame_y >= s->avctx->height ||
+        frame_height > s->avctx->height ||
+        frame_y + frame_height > s->avctx->height)
+        return;
 
     if ((frame_width == s->avctx->width && frame_height == s->avctx->height) &&
         (frame_x || frame_y)) {
@@ -216,8 +246,9 @@ static void vmd_decode(VmdVideoContext *s)
 
     /* if only a certain region will be updated, copy the entire previous
      * frame before the decode */
-    if (frame_x || frame_y || (frame_width != s->avctx->width) ||
-        (frame_height != s->avctx->height)) {
+    if (s->prev_frame.data[0] &&
+        (frame_x || frame_y || (frame_width != s->avctx->width) ||
+        (frame_height != s->avctx->height))) {
 
         memcpy(s->frame.data[0], s->prev_frame.data[0],
             s->avctx->height * s->frame.linesize[0]);
@@ -225,6 +256,8 @@ static void vmd_decode(VmdVideoContext *s)
 
     /* check if there is a new palette */
     if (s->buf[15] & 0x02) {
+        if (p_end - p < 2 + 3 * PALETTE_COUNT)
+            return;
         p += 2;
         palette32 = (unsigned int *)s->palette;
         for (i = 0; i < PALETTE_COUNT; i++) {
@@ -233,16 +266,17 @@ static void vmd_decode(VmdVideoContext *s)
             b = *p++ * 4;
             palette32[i] = (r << 16) | (g << 8) | (b);
         }
-        s->size -= (256 * 3 + 2);
     }
-    if (s->size >= 0) {
+    if (p < p_end) {
         /* originally UnpackFrame in VAG's code */
         pb = p;
+        pb_end = p_end;
         meth = *pb++;
         if (meth & 0x80) {
-            lz_unpack(pb, s->unpack_buffer, s->unpack_buffer_size);
+            lz_unpack(pb, p_end - pb, s->unpack_buffer, s->unpack_buffer_size);
             meth &= 0x7F;
             pb = s->unpack_buffer;
+            pb_end = s->unpack_buffer + s->unpack_buffer_size;
         }
 
         dp = &s->frame.data[0][frame_y * s->frame.linesize[0] + frame_x];
@@ -252,17 +286,19 @@ static void vmd_decode(VmdVideoContext *s)
             for (i = 0; i < frame_height; i++) {
                 ofs = 0;
                 do {
+                    if (pb_end - pb < 1)
+                        return;
                     len = *pb++;
                     if (len & 0x80) {
                         len = (len & 0x7F) + 1;
-                        if (ofs + len > frame_width)
+                        if (ofs + len > frame_width || pb_end - pb < len)
                             return;
                         memcpy(&dp[ofs], pb, len);
                         pb += len;
                         ofs += len;
                     } else {
                         /* interframe pixel copy */
-                        if (ofs + len + 1 > frame_width)
+                        if (ofs + len + 1 > frame_width || !s->prev_frame.data[0])
                             return;
                         memcpy(&dp[ofs], &pp[ofs], len + 1);
                         ofs += len + 1;
@@ -280,6 +316,8 @@ static void vmd_decode(VmdVideoContext *s)
 
         case 2:
             for (i = 0; i < frame_height; i++) {
+                if (pb_end -pb < frame_width)
+                    return;
                 memcpy(dp, pb, frame_width);
                 pb += frame_width;
                 dp += s->frame.linesize[0];
@@ -291,18 +329,25 @@ static void vmd_decode(VmdVideoContext *s)
             for (i = 0; i < frame_height; i++) {
                 ofs = 0;
                 do {
+                    if (pb_end - pb < 1)
+                        return;
                     len = *pb++;
                     if (len & 0x80) {
                         len = (len & 0x7F) + 1;
+                        if (pb_end - pb < 1)
+                            return;
                         if (*pb++ == 0xFF)
-                            len = rle_unpack(pb, &dp[ofs], len, frame_width - ofs);
-                        else
+                            len = rle_unpack(pb, pb_end - pb, len, &dp[ofs], frame_width - ofs);
+                        else {
+                        if (pb_end - pb < len)
+                            return;
                             memcpy(&dp[ofs], pb, len);
+                        }
                         pb += len;
                         ofs += len;
                     } else {
                         /* interframe pixel copy */
-                        if (ofs + len + 1 > frame_width)
+                        if (ofs + len + 1 > frame_width || !s->prev_frame.data[0])
                             return;
                         memcpy(&dp[ofs], &pp[ofs], len + 1);
                         ofs += len + 1;
@@ -523,7 +568,10 @@ static int vmdaudio_decode_frame(AVCodecContext *avctx,
 
     silent_chunks = 0;
     if (block_type == BLOCK_TYPE_INITIAL) {
-        uint32_t flags = AV_RB32(buf);
+        uint32_t flags;
+        if (buf_size < 4)
+            return -1;
+        flags = AV_RB32(buf);
         silent_chunks  = av_popcount(flags);
         buf      += 4;
         buf_size -= 4;