Merge commit 'dbaf3f7b0bc9e99dff8e06bd29fcb3e84eebfe7c' into release/1.1

* commit 'dbaf3f7b0bc9e99dff8e06bd29fcb3e84eebfe7c':
  vmd: drop incomplete chunks and spurious samples
  vmd: return meaningful errors

Conflicts:
	libavcodec/vmdav.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 31 insertions, 12 deletions

diff --git a/libavcodec/vmdav.c b/libavcodec/vmdav.c
index ca51ccca6f..60ac5794fb 100644
--- a/libavcodec/vmdav.c
+++ b/libavcodec/vmdav.c
@@ -43,6 +43,7 @@
 #include <stdlib.h>
 #include <string.h>
 
+#include "libavutil/avassert.h"
 #include "libavutil/channel_layout.h"
 #include "libavutil/common.h"
 #include "libavutil/intreadwrite.h"
@@ -192,7 +193,7 @@ static int rle_unpack(const unsigned char *src, unsigned char *dest,
     return bytestream2_tell(&gb);
 }
 
-static void vmd_decode(VmdVideoContext *s)
+static int vmd_decode(VmdVideoContext *s)
 {
     int i;
     unsigned int *palette32;
@@ -216,13 +217,21 @@ static void vmd_decode(VmdVideoContext *s)
     if (frame_x < 0 || frame_width < 0 ||
         frame_x >= s->avctx->width ||
         frame_width > s->avctx->width ||
-        frame_x + frame_width > s->avctx->width)
-        return;
+        frame_x + frame_width > s->avctx->width) {
+        av_log(s->avctx, AV_LOG_ERROR,
+               "Invalid horizontal range %d-%d\n",
+               frame_x, frame_width);
+        return AVERROR_INVALIDDATA;
+    }
     if (frame_y < 0 || frame_height < 0 ||
         frame_y >= s->avctx->height ||
         frame_height > s->avctx->height ||
-        frame_y + frame_height > s->avctx->height)
-        return;
+        frame_y + frame_height > s->avctx->height) {
+        av_log(s->avctx, AV_LOG_ERROR,
+               "Invalid vertical range %d-%d\n",
+               frame_x, frame_width);
+        return AVERROR_INVALIDDATA;
+    }
 
     if ((frame_width == s->avctx->width && frame_height == s->avctx->height) &&
         (frame_x || frame_y)) {
@@ -256,13 +265,16 @@ static void vmd_decode(VmdVideoContext *s)
                 palette32[i] = 0xFFU << 24 | (r << 16) | (g << 8) | (b);
                 palette32[i] |= palette32[i] >> 6 & 0x30303;
             }
+        } else {
+            av_log(s->avctx, AV_LOG_ERROR, "Incomplete palette\n");
+            return AVERROR_INVALIDDATA;
         }
     }
     if (s->size > 0) {
         /* originally UnpackFrame in VAG's code */
         bytestream2_init(&gb, gb.buffer, s->buf + s->size - gb.buffer);
         if (bytestream2_get_bytes_left(&gb) < 1)
-            return;
+            return AVERROR_INVALIDDATA;
         meth = bytestream2_get_byteu(&gb);
         if (meth & 0x80) {
             lz_unpack(gb.buffer, bytestream2_get_bytes_left(&gb),
@@ -282,13 +294,13 @@ static void vmd_decode(VmdVideoContext *s)
                     if (len & 0x80) {
                         len = (len & 0x7F) + 1;
                         if (ofs + len > frame_width || bytestream2_get_bytes_left(&gb) < len)
-                            return;
+                            return AVERROR_INVALIDDATA;
                         bytestream2_get_buffer(&gb, &dp[ofs], len);
                         ofs += len;
                     } else {
                         /* interframe pixel copy */
                         if (ofs + len + 1 > frame_width || !s->prev_frame.data[0])
-                            return;
+                            return AVERROR_INVALIDDATA;
                         memcpy(&dp[ofs], &pp[ofs], len + 1);
                         ofs += len + 1;
                     }
@@ -296,7 +308,7 @@ static void vmd_decode(VmdVideoContext *s)
                 if (ofs > frame_width) {
                     av_log(s->avctx, AV_LOG_ERROR, "offset > width (%d > %d)\n",
                         ofs, frame_width);
-                    break;
+                    return AVERROR_INVALIDDATA;
                 }
                 dp += s->frame.linesize[0];
                 pp += s->prev_frame.linesize[0];
@@ -328,7 +340,7 @@ static void vmd_decode(VmdVideoContext *s)
                     } else {
                         /* interframe pixel copy */
                         if (ofs + len + 1 > frame_width || !s->prev_frame.data[0])
-                            return;
+                            return AVERROR_INVALIDDATA;
                         memcpy(&dp[ofs], &pp[ofs], len + 1);
                         ofs += len + 1;
                     }
@@ -336,6 +348,7 @@ static void vmd_decode(VmdVideoContext *s)
                 if (ofs > frame_width) {
                     av_log(s->avctx, AV_LOG_ERROR, "offset > width (%d > %d)\n",
                         ofs, frame_width);
+                    return AVERROR_INVALIDDATA;
                 }
                 dp += s->frame.linesize[0];
                 pp += s->prev_frame.linesize[0];
@@ -343,6 +356,7 @@ static void vmd_decode(VmdVideoContext *s)
             break;
         }
     }
+    return 0;
 }
 
 static av_cold int vmdvideo_decode_init(AVCodecContext *avctx)
@@ -399,7 +413,7 @@ static int vmdvideo_decode_frame(AVCodecContext *avctx,
     s->size = buf_size;
 
     if (buf_size < 16)
-        return buf_size;
+        return AVERROR_INVALIDDATA;
 
     s->frame.reference = 3;
     if (ff_get_buffer(avctx, &s->frame)) {
@@ -576,6 +590,9 @@ static int vmdaudio_decode_frame(AVCodecContext *avctx, void *data,
     /* ensure output buffer is large enough */
     audio_chunks = buf_size / s->chunk_size;
 
+    /* drop incomplete chunks */
+    buf_size     = audio_chunks * s->chunk_size;
+
     /* get output buffer */
     s->frame.nb_samples = ((silent_chunks + audio_chunks) * avctx->block_align) / avctx->channels;
     if ((ret = ff_get_buffer(avctx, &s->frame)) < 0) {
@@ -587,7 +604,8 @@ static int vmdaudio_decode_frame(AVCodecContext *avctx, void *data,
 
     /* decode silent chunks */
     if (silent_chunks > 0) {
-        int silent_size = avctx->block_align * silent_chunks;
+        int silent_size = FFMIN(avctx->block_align * silent_chunks,
+                                s->frame.nb_samples * avctx->channels);
         if (s->out_bps == 2) {
             memset(output_samples_s16, 0x00, silent_size * 2);
             output_samples_s16 += silent_size;
@@ -600,6 +618,7 @@ static int vmdaudio_decode_frame(AVCodecContext *avctx, void *data,
     /* decode audio chunks */
     if (audio_chunks > 0) {
         buf_end = buf + buf_size;
+        av_assert0((buf_size & (avctx->channels > 1)) == 0);
         while (buf_end - buf >= s->chunk_size) {
             if (s->out_bps == 2) {
                 decode_audio_s16(output_samples_s16, buf, s->chunk_size,