Merge commit 'b19eafa2b930ee40abfde6d1f026b7fa5591c4dc' into release/0.10

* commit 'b19eafa2b930ee40abfde6d1f026b7fa5591c4dc':
  eacmv: Make sure a reference frame exists before referencing it
  mpeg4videodec: Check the width/height in mpeg4_decode_sprite_trajectory
  ivi_common: Make sure color planes have been initialized
  oggparseogm: Convert to use bytestream2
  rv34: Check the return value from ff_rv34_decode_init
  matroskadec: Verify realaudio codec parameters
  mace: Make sure that the channel count is set to a valid value
  svq3: Check for any negative return value from ff_h264_check_intra_pred_mode
  vp3: Check the framerate for validity
  cavsdec: Make sure a sequence header has been decoded before decoding pictures
  sierravmd: Do sanity checking of frame sizes
  omadec: Properly check lengths before incrementing the position
  mpc8: Make sure the first stream exists before parsing the seek table

Conflicts:
	libavcodec/eacmv.c
	libavcodec/mpeg4videodec.c
	libavformat/omadec.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

14 files changed, 90 insertions, 49 deletions

diff --git a/libavcodec/cavsdec.c b/libavcodec/cavsdec.c
index 0b617061eb..221ead6d18 100644
--- a/libavcodec/cavsdec.c
+++ b/libavcodec/cavsdec.c
@@ -468,6 +468,11 @@ static int decode_pic(AVSContext *h) {
     int skip_count = -1;
     enum cavs_mb mb_type;
 
+    if (!h->top_qp) {
+        av_log(h, AV_LOG_ERROR, "No sequence header decoded yet\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     if (!s->context_initialized) {
         s->avctx->idct_algo = FF_IDCT_CAVS;
         if (MPV_common_init(s) < 0)
diff --git a/libavcodec/eacmv.c b/libavcodec/eacmv.c
index 8a8d6a9cef..d7b8562aca 100644
--- a/libavcodec/eacmv.c
+++ b/libavcodec/eacmv.c
@@ -112,8 +112,8 @@ static void cmv_decode_inter(CmvContext * s, const uint8_t *buf, const uint8_t *
             int yoffset = ((buf[i] >> 4)) - 7;
             if (s->last_frame.data[0])
                 cmv_motcomp(s->frame.data[0], s->frame.linesize[0],
-                          s->last_frame.data[0], s->last_frame.linesize[0],
-                          x*4, y*4, xoffset, yoffset, s->avctx->width, s->avctx->height);
+                            s->last_frame.data[0], s->last_frame.linesize[0],
+                            x*4, y*4, xoffset, yoffset, s->avctx->width, s->avctx->height);
         }
         i++;
     }
diff --git a/libavcodec/ivi_common.c b/libavcodec/ivi_common.c
index ee16b2a28c..bd590af2f9 100644
--- a/libavcodec/ivi_common.c
+++ b/libavcodec/ivi_common.c
@@ -894,6 +894,11 @@ int ff_ivi_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         return AVERROR_PATCHWELCOME;
     }
 
+    if (!ctx->planes[0].bands) {
+        av_log(avctx, AV_LOG_ERROR, "Color planes not initialized yet\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     ctx->switch_buffers(ctx);
 
     //{ START_TIMER;
diff --git a/libavcodec/mace.c b/libavcodec/mace.c
index ffa11ad80d..fcd49d957f 100644
--- a/libavcodec/mace.c
+++ b/libavcodec/mace.c
@@ -231,8 +231,8 @@ static av_cold int mace_decode_init(AVCodecContext * avctx)
 {
     MACEContext *ctx = avctx->priv_data;
 
-    if (avctx->channels > 2)
-        return -1;
+    if (avctx->channels > 2 || avctx->channels < 1)
+        return AVERROR(EINVAL);
     avctx->sample_fmt = AV_SAMPLE_FMT_S16;
 
     avcodec_get_frame_defaults(&ctx->frame);
diff --git a/libavcodec/mpeg4videodec.c b/libavcodec/mpeg4videodec.c
index 140fce8297..859b04f552 100644
--- a/libavcodec/mpeg4videodec.c
+++ b/libavcodec/mpeg4videodec.c
@@ -160,7 +160,7 @@ static inline int mpeg4_is_resync(MpegEncContext *s){
     return 0;
 }
 
-static int mpeg4_decode_sprite_trajectory(MpegEncContext * s, GetBitContext *gb)
+static int mpeg4_decode_sprite_trajectory(MpegEncContext *s, GetBitContext *gb)
 {
     int i;
     int a= 2<<s->sprite_warping_accuracy;
@@ -176,8 +176,8 @@ static int mpeg4_decode_sprite_trajectory(MpegEncContext * s, GetBitContext *gb)
     int h= s->height;
     int min_ab;
 
-    if(w<=0 || h<=0)
-        return -1;
+    if (w <= 0 || h <= 0)
+        return AVERROR_INVALIDDATA;
 
     for(i=0; i<s->num_sprite_warping_points; i++){
         int length;
@@ -415,8 +415,8 @@ int mpeg4_decode_video_packet_header(MpegEncContext *s)
             skip_bits(&s->gb, 3); /* intra dc vlc threshold */
 //FIXME don't just ignore everything
             if(s->pict_type == AV_PICTURE_TYPE_S && s->vol_sprite_usage==GMC_SPRITE){
-                if(mpeg4_decode_sprite_trajectory(s, &s->gb) < 0)
-                    return -1;
+                if (mpeg4_decode_sprite_trajectory(s, &s->gb) < 0)
+                    return AVERROR_INVALIDDATA;
                 av_log(s->avctx, AV_LOG_ERROR, "untested\n");
             }
 
@@ -2056,8 +2056,8 @@ static int decode_vop_header(MpegEncContext *s, GetBitContext *gb){
      }
 
      if(s->pict_type == AV_PICTURE_TYPE_S && (s->vol_sprite_usage==STATIC_SPRITE || s->vol_sprite_usage==GMC_SPRITE)){
-         if(mpeg4_decode_sprite_trajectory(s, gb) < 0)
-             return -1;
+         if (mpeg4_decode_sprite_trajectory(s, gb) < 0)
+             return AVERROR_INVALIDDATA;
          if(s->sprite_brightness_change) av_log(s->avctx, AV_LOG_ERROR, "sprite_brightness_change not supported\n");
          if(s->vol_sprite_usage==STATIC_SPRITE) av_log(s->avctx, AV_LOG_ERROR, "static sprite not supported\n");
      }
diff --git a/libavcodec/rv30.c b/libavcodec/rv30.c
index e60c93db4b..2a44363e4f 100644
--- a/libavcodec/rv30.c
+++ b/libavcodec/rv30.c
@@ -249,9 +249,11 @@ static void rv30_loop_filter(RV34DecContext *r, int row)
 static av_cold int rv30_decode_init(AVCodecContext *avctx)
 {
     RV34DecContext *r = avctx->priv_data;
+    int ret;
 
     r->rv30 = 1;
-    ff_rv34_decode_init(avctx);
+    if ((ret = ff_rv34_decode_init(avctx)) < 0)
+        return ret;
     if(avctx->extradata_size < 2){
         av_log(avctx, AV_LOG_ERROR, "Extradata is too small.\n");
         return -1;
diff --git a/libavcodec/rv40.c b/libavcodec/rv40.c
index 98e94eaa97..c46b7ef7e1 100644
--- a/libavcodec/rv40.c
+++ b/libavcodec/rv40.c
@@ -544,9 +544,11 @@ static void rv40_loop_filter(RV34DecContext *r, int row)
 static av_cold int rv40_decode_init(AVCodecContext *avctx)
 {
     RV34DecContext *r = avctx->priv_data;
+    int ret;
 
     r->rv30 = 0;
-    ff_rv34_decode_init(avctx);
+    if ((ret = ff_rv34_decode_init(avctx)) < 0)
+        return ret;
     if(!aic_top_vlc.bits)
         rv40_init_tables();
     r->parse_slice_header = rv40_parse_slice_header;
diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c
index 552c0bf141..6f5a2a91ac 100644
--- a/libavcodec/svq3.c
+++ b/libavcodec/svq3.c
@@ -613,9 +613,9 @@ static int svq3_decode_mb(SVQ3Context *svq3, unsigned int mb_type)
         dir = i_mb_type_info[mb_type - 8].pred_mode;
         dir = (dir >> 1) ^ 3*(dir & 1) ^ 1;
 
-        if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir, 0)) == -1){
-            av_log(h->s.avctx, AV_LOG_ERROR, "check_intra_pred_mode = -1\n");
-            return -1;
+        if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir, 0)) < 0) {
+            av_log(h->s.avctx, AV_LOG_ERROR, "ff_h264_check_intra_pred_mode < 0\n");
+            return h->intra16x16_pred_mode;
         }
 
         cbp = i_mb_type_info[mb_type - 8].cbp;
diff --git a/libavcodec/vp3.c b/libavcodec/vp3.c
index f4f87d3349..10c5b8c36f 100644
--- a/libavcodec/vp3.c
+++ b/libavcodec/vp3.c
@@ -2150,6 +2150,10 @@ static int theora_decode_header(AVCodecContext *avctx, GetBitContext *gb)
     fps.num = get_bits_long(gb, 32);
     fps.den = get_bits_long(gb, 32);
     if (fps.num && fps.den) {
+        if (fps.num < 0 || fps.den < 0) {
+            av_log(avctx, AV_LOG_ERROR, "Invalid framerate\n");
+            return AVERROR_INVALIDDATA;
+        }
         av_reduce(&avctx->time_base.num, &avctx->time_base.den,
                   fps.den, fps.num, 1<<30);
     }
diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c
index 5e52f86e81..45bafe5ec4 100644
--- a/libavformat/matroskadec.c
+++ b/libavformat/matroskadec.c
@@ -1536,6 +1536,10 @@ static int matroska_read_header(AVFormatContext *s, AVFormatParameters *ap)
             track->audio.sub_packet_h    = avio_rb16(&b);
             track->audio.frame_size      = avio_rb16(&b);
             track->audio.sub_packet_size = avio_rb16(&b);
+            if (flavor <= 0 || track->audio.coded_framesize <= 0 ||
+                track->audio.sub_packet_h <= 0 || track->audio.frame_size <= 0 ||
+                track->audio.sub_packet_size <= 0)
+                return AVERROR_INVALIDDATA;
             track->audio.buf = av_malloc(track->audio.frame_size * track->audio.sub_packet_h);
             if (codec_id == CODEC_ID_RA_288) {
                 st->codec->block_align = track->audio.coded_framesize;
diff --git a/libavformat/mpc8.c b/libavformat/mpc8.c
index 4117d03e5a..a3fc1be894 100644
--- a/libavformat/mpc8.c
+++ b/libavformat/mpc8.c
@@ -137,6 +137,11 @@ static void mpc8_parse_seektable(AVFormatContext *s, int64_t off)
     int i, t, seekd;
     GetBitContext gb;
 
+    if (s->nb_streams == 0) {
+        av_log(s, AV_LOG_ERROR, "No stream added before parsing seek table\n");
+        return;
+    }
+
     avio_seek(s->pb, off, SEEK_SET);
     mpc8_get_chunk_header(s->pb, &tag, &size);
     if(tag != TAG_SEEKTABLE){
diff --git a/libavformat/oggparseogm.c b/libavformat/oggparseogm.c
index 0a8a7c6bd4..c761bbd7db 100644
--- a/libavformat/oggparseogm.c
+++ b/libavformat/oggparseogm.c
@@ -37,62 +37,64 @@ ogm_header(AVFormatContext *s, int idx)
     struct ogg *ogg = s->priv_data;
     struct ogg_stream *os = ogg->streams + idx;
     AVStream *st = s->streams[idx];
-    const uint8_t *p = os->buf + os->pstart;
+    GetByteContext p;
     uint64_t time_unit;
     uint64_t spu;
 
-    if(!(*p & 1))
+    bytestream2_init(&p, os->buf + os->pstart, os->psize);
+    if (!(bytestream2_peek_byte(&p) & 1))
         return 0;
 
-    if(*p == 1) {
-        p++;
+    if (bytestream2_peek_byte(&p) == 1) {
+        bytestream2_skip(&p, 1);
 
-        if(*p == 'v'){
+        if (bytestream2_peek_byte(&p) == 'v'){
             int tag;
             st->codec->codec_type = AVMEDIA_TYPE_VIDEO;
-            p += 8;
-            tag = bytestream_get_le32(&p);
+            bytestream2_skip(&p, 8);
+            tag = bytestream2_get_le32(&p);
             st->codec->codec_id = ff_codec_get_id(ff_codec_bmp_tags, tag);
             st->codec->codec_tag = tag;
-        } else if (*p == 't') {
+        } else if (bytestream2_peek_byte(&p) == 't') {
             st->codec->codec_type = AVMEDIA_TYPE_SUBTITLE;
             st->codec->codec_id = CODEC_ID_TEXT;
-            p += 12;
+            bytestream2_skip(&p, 12);
         } else {
-            uint8_t acid[5];
+            uint8_t acid[5] = { 0 };
             int cid;
             st->codec->codec_type = AVMEDIA_TYPE_AUDIO;
-            p += 8;
-            bytestream_get_buffer(&p, acid, 4);
+            bytestream2_skip(&p, 8);
+            bytestream2_get_buffer(&p, acid, 4);
             acid[4] = 0;
             cid = strtol(acid, NULL, 16);
             st->codec->codec_id = ff_codec_get_id(ff_codec_wav_tags, cid);
             st->need_parsing = AVSTREAM_PARSE_FULL;
         }
 
-        p += 4;                     /* useless size field */
+        bytestream2_skip(&p, 4);    /* useless size field */
 
-        time_unit   = bytestream_get_le64(&p);
-        spu         = bytestream_get_le64(&p);
-        p += 4;                     /* default_len */
-        p += 8;                     /* buffersize + bits_per_sample */
+        time_unit   = bytestream2_get_le64(&p);
+        spu         = bytestream2_get_le64(&p);
+        bytestream2_skip(&p, 4);    /* default_len */
+        bytestream2_skip(&p, 8);    /* buffersize + bits_per_sample */
 
         if(st->codec->codec_type == AVMEDIA_TYPE_VIDEO){
-            st->codec->width = bytestream_get_le32(&p);
-            st->codec->height = bytestream_get_le32(&p);
+            st->codec->width = bytestream2_get_le32(&p);
+            st->codec->height = bytestream2_get_le32(&p);
             st->codec->time_base.den = spu * 10000000;
             st->codec->time_base.num = time_unit;
             avpriv_set_pts_info(st, 64, st->codec->time_base.num, st->codec->time_base.den);
         } else {
-            st->codec->channels = bytestream_get_le16(&p);
-            p += 2;                 /* block_align */
-            st->codec->bit_rate = bytestream_get_le32(&p) * 8;
+            st->codec->channels = bytestream2_get_le16(&p);
+            bytestream2_skip(&p, 2); /* block_align */
+            st->codec->bit_rate = bytestream2_get_le32(&p) * 8;
             st->codec->sample_rate = spu * 10000000 / time_unit;
             avpriv_set_pts_info(st, 64, 1, st->codec->sample_rate);
         }
-    } else if (*p == 3) {
-        if (os->psize > 8)
-            ff_vorbis_comment(s, &st->metadata, p+7, os->psize-8);
+    } else if (bytestream2_peek_byte(&p) == 3) {
+        bytestream2_skip(&p, 7);
+        if (bytestream2_get_bytes_left(&p) > 1)
+            ff_vorbis_comment(s, &st->metadata, p.buffer, bytestream2_get_bytes_left(&p) - 1);
     }
 
     return 1;
diff --git a/libavformat/omadec.c b/libavformat/omadec.c
index 1df2244607..93c661e615 100644
--- a/libavformat/omadec.c
+++ b/libavformat/omadec.c
@@ -173,7 +173,11 @@ static int nprobe(AVFormatContext *s, uint8_t *enc_header, unsigned size,
     if(taglen + (((uint64_t)datalen)<<4) + 44 > size)
         return -1;
 
-    pos += 44 + taglen;
+    pos += 44;
+    if (size - pos < taglen)
+        return -1;
+
+    pos += taglen;
 
     if (datalen << 4 > size - pos)
         return -1;
diff --git a/libavformat/sierravmd.c b/libavformat/sierravmd.c
index 6fb7d8a700..560ffe6792 100644
--- a/libavformat/sierravmd.c
+++ b/libavformat/sierravmd.c
@@ -89,7 +89,7 @@ static int vmd_read_header(AVFormatContext *s,
     unsigned char *raw_frame_table;
     int raw_frame_table_size;
     int64_t current_offset;
-    int i, j;
+    int i, j, ret;
     unsigned int total_frames;
     int64_t current_audio_pts = 0;
     unsigned char chunk[BYTES_PER_FRAME_RECORD];
@@ -170,15 +170,13 @@ static int vmd_read_header(AVFormatContext *s,
     raw_frame_table = av_malloc(raw_frame_table_size);
     vmd->frame_table = av_malloc((vmd->frame_count * vmd->frames_per_block + sound_buffers) * sizeof(vmd_frame));
     if (!raw_frame_table || !vmd->frame_table) {
-        av_free(raw_frame_table);
-        av_free(vmd->frame_table);
-        return AVERROR(ENOMEM);
+        ret = AVERROR(ENOMEM);
+        goto error;
     }
     if (avio_read(pb, raw_frame_table, raw_frame_table_size) !=
         raw_frame_table_size) {
-        av_free(raw_frame_table);
-        av_free(vmd->frame_table);
-        return AVERROR(EIO);
+        ret = AVERROR(EIO);
+        goto error;
     }
 
     total_frames = 0;
@@ -194,6 +192,11 @@ static int vmd_read_header(AVFormatContext *s,
             avio_read(pb, chunk, BYTES_PER_FRAME_RECORD);
             type = chunk[0];
             size = AV_RL32(&chunk[2]);
+            if (size > INT_MAX / 2) {
+                av_log(s, AV_LOG_ERROR, "Invalid frame size\n");
+                ret = AVERROR_INVALIDDATA;
+                goto error;
+            }
             if(!size && type != 1)
                 continue;
             switch(type) {
@@ -230,6 +233,11 @@ static int vmd_read_header(AVFormatContext *s,
     vmd->frame_count = total_frames;
 
     return 0;
+
+error:
+    av_free(raw_frame_table);
+    av_free(vmd->frame_table);
+    return ret;
 }
 
 static int vmd_read_packet(AVFormatContext *s,