Merge remote-tracking branch 'qatar/release/0.5' into release/0.5

* qatar/release/0.5: (21 commits)
  vp6: properly fail on unsupported feature
  vp56: release frames on error
  shorten: Use separate pointers for the allocated memory for decoded samples.
  shorten: check for realloc failure
  h264: check context state before decoding slice data partitions
  oggdec: check memory allocation
  Fix uninitialized reads on malformed ogg files.
  lavf: avoid integer overflow in ff_compute_frame_duration()
  yuv4mpeg: reject unsupported codecs
  tiffenc: Check av_malloc() results.
  mpegaudiodec: fix short_start calculation
  h264: avoid stuck buffer pointer in decode_nal_units
  yuv4mpeg: return proper error codes.
  avidec: return 0, not packet size from read_packet().
  cavsdec: check for changing w/h.
  avidec: use actually read size instead of requested size
  bytestream: add a new set of bytestream functions with overread checking
  avsdec: Set dimensions instead of relying on the demuxer.
  lavfi: avfilter_merge_formats: handle case where inputs are same
  bmpdec: only initialize palette for pal8.
  ...

Merged-by: Michael Niedermayer <michaelni@gmx.at>

15 files changed, 159 insertions, 35 deletions

diff --git a/libavcodec/avs.c b/libavcodec/avs.c
index 3b29c853b4..e8a55fdddd 100644
--- a/libavcodec/avs.c
+++ b/libavcodec/avs.c
@@ -145,6 +145,7 @@ avs_decode_frame(AVCodecContext * avctx,
 static av_cold int avs_decode_init(AVCodecContext * avctx)
 {
     avctx->pix_fmt = PIX_FMT_PAL8;
+    avcodec_set_dimensions(avctx, 318, 198);
     return 0;
 }
 
diff --git a/libavcodec/bmp.c b/libavcodec/bmp.c
index 14c070da8f..30bae8c7d0 100644
--- a/libavcodec/bmp.c
+++ b/libavcodec/bmp.c
@@ -217,9 +217,6 @@ static int bmp_decode_frame(AVCodecContext *avctx,
     if(comp == BMP_RLE4 || comp == BMP_RLE8)
         memset(p->data[0], 0, avctx->height * p->linesize[0]);
 
-    if(depth == 4 || depth == 8)
-        memset(p->data[1], 0, 1024);
-
     if(height > 0){
         ptr = p->data[0] + (avctx->height - 1) * p->linesize[0];
         linesize = -p->linesize[0];
@@ -229,6 +226,9 @@ static int bmp_decode_frame(AVCodecContext *avctx,
     }
 
     if(avctx->pix_fmt == PIX_FMT_PAL8){
+
+        memset(p->data[1], 0, 1024);
+
         buf = buf0 + 14 + ihsize; //palette location
         if((hsize-ihsize-14)>>depth < 4){ // OS/2 bitmap, 3 bytes per palette entry
             for(i = 0; i < (1 << depth); i++)
diff --git a/libavcodec/bytestream.h b/libavcodec/bytestream.h
index b56f6ce743..7ca36f8ad3 100644
--- a/libavcodec/bytestream.h
+++ b/libavcodec/bytestream.h
@@ -26,6 +26,10 @@
 #include "libavutil/common.h"
 #include "libavutil/intreadwrite.h"
 
+typedef struct {
+    const uint8_t *buffer, *buffer_end;
+} GetByteContext;
+
 #define DEF_T(type, name, bytes, read, write)                             \
 static av_always_inline type bytestream_get_ ## name(const uint8_t **b){\
     (*b) += bytes;\
@@ -34,6 +38,18 @@ static av_always_inline type bytestream_get_ ## name(const uint8_t **b){\
 static av_always_inline void bytestream_put_ ##name(uint8_t **b, const type value){\
     write(*b, value);\
     (*b) += bytes;\
+}\
+static av_always_inline type bytestream2_get_ ## name(GetByteContext *g)\
+{\
+    if (g->buffer_end - g->buffer < bytes)\
+        return 0;\
+    return bytestream_get_ ## name(&g->buffer);\
+}\
+static av_always_inline type bytestream2_peek_ ## name(GetByteContext *g)\
+{\
+    if (g->buffer_end - g->buffer < bytes)\
+        return 0;\
+    return read(g->buffer);\
 }
 
 #define DEF(name, bytes, read, write) \
@@ -55,6 +71,34 @@ DEF  (byte, 1, AV_RB8 , AV_WB8 )
 #undef DEF64
 #undef DEF_T
 
+static av_always_inline void bytestream2_init(GetByteContext *g,
+                                              const uint8_t *buf, int buf_size)
+{
+    g->buffer =  buf;
+    g->buffer_end = buf + buf_size;
+}
+
+static av_always_inline unsigned int bytestream2_get_bytes_left(GetByteContext *g)
+{
+    return g->buffer_end - g->buffer;
+}
+
+static av_always_inline void bytestream2_skip(GetByteContext *g,
+                                              unsigned int size)
+{
+    g->buffer += FFMIN(g->buffer_end - g->buffer, size);
+}
+
+static av_always_inline unsigned int bytestream2_get_buffer(GetByteContext *g,
+                                                            uint8_t *dst,
+                                                            unsigned int size)
+{
+    int size2 = FFMIN(g->buffer_end - g->buffer, size);
+    memcpy(dst, g->buffer, size2);
+    g->buffer += size2;
+    return size2;
+}
+
 static av_always_inline unsigned int bytestream_get_buffer(const uint8_t **b, uint8_t *dst, unsigned int size)
 {
     memcpy(dst, *b, size);
diff --git a/libavcodec/cavsdec.c b/libavcodec/cavsdec.c
index 25b97bfde8..fae8879476 100644
--- a/libavcodec/cavsdec.c
+++ b/libavcodec/cavsdec.c
@@ -599,12 +599,21 @@ static int decode_pic(AVSContext *h) {
 static int decode_seq_header(AVSContext *h) {
     MpegEncContext *s = &h->s;
     int frame_rate_code;
+    int width, height;
 
     h->profile =         get_bits(&s->gb,8);
     h->level =           get_bits(&s->gb,8);
     skip_bits1(&s->gb); //progressive sequence
-    s->width =           get_bits(&s->gb,14);
-    s->height =          get_bits(&s->gb,14);
+
+    width  = get_bits(&s->gb, 14);
+    height = get_bits(&s->gb, 14);
+    if ((s->width || s->height) && (s->width != width || s->height != height)) {
+        av_log(s, AV_LOG_ERROR, "Width/height changing in CAVS is unsupported");
+        return AVERROR_PATCHWELCOME;
+    }
+    s->width  = width;
+    s->height = height;
+
     skip_bits(&s->gb,2); //chroma format
     skip_bits(&s->gb,3); //sample_precision
     h->aspect_ratio =    get_bits(&s->gb,4);
diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 48c8028c10..099f5f1a3a 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -7459,7 +7459,11 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size){
                     break;
             }
 
-            if(buf_index+3 >= buf_size) break;
+
+            if (buf_index + 3 >= buf_size) {
+                buf_index = buf_size;
+                break;
+            }
 
             buf_index+=3;
         }
@@ -7553,6 +7557,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size){
             hx->inter_gb_ptr= &hx->inter_gb;
 
             if(hx->redundant_pic_count==0 && hx->intra_gb_ptr && hx->s.data_partitioning
+               && s->current_picture_ptr
                && s->context_initialized
                && s->hurry_up < 5
                && (avctx->skip_frame < AVDISCARD_NONREF || hx->nal_ref_idc)
diff --git a/libavcodec/mpegaudiodec.c b/libavcodec/mpegaudiodec.c
index e7bbd5d541..c95a571f72 100644
--- a/libavcodec/mpegaudiodec.c
+++ b/libavcodec/mpegaudiodec.c
@@ -173,7 +173,7 @@ void ff_compute_band_indexes(MPADecodeContext *s, GranuleDef *g){
             else
                 g->long_end = 4; /* 8000 Hz */
 
-            g->short_start = 2 + (s->sample_rate_index != 8);
+            g->short_start = 3;
         } else {
             g->long_end = 0;
             g->short_start = 0;
diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c
index 6aacd5c6e4..b39c3ab632 100644
--- a/libavcodec/shorten.c
+++ b/libavcodec/shorten.c
@@ -82,6 +82,7 @@ typedef struct ShortenContext {
     int channels;
 
     int32_t *decoded[MAX_CHANNELS];
+    int32_t *decoded_base[MAX_CHANNELS];
     int32_t *offset[MAX_CHANNELS];
     uint8_t *bitstream;
     int bitstream_size;
@@ -112,6 +113,8 @@ static av_cold int shorten_decode_init(AVCodecContext * avctx)
 static int allocate_buffers(ShortenContext *s)
 {
     int i, chan;
+    void *tmp_ptr;
+
     for (chan=0; chan<s->channels; chan++) {
         if(FFMAX(1, s->nmean) >= UINT_MAX/sizeof(int32_t)){
             av_log(s->avctx, AV_LOG_ERROR, "nmean too large\n");
@@ -122,12 +125,19 @@ static int allocate_buffers(ShortenContext *s)
             return -1;
         }
 
-        s->offset[chan] = av_realloc(s->offset[chan], sizeof(int32_t)*FFMAX(1, s->nmean));
+        tmp_ptr = av_realloc(s->offset[chan], sizeof(int32_t)*FFMAX(1, s->nmean));
+        if (!tmp_ptr)
+            return AVERROR(ENOMEM);
+        s->offset[chan] = tmp_ptr;
 
-        s->decoded[chan] = av_realloc(s->decoded[chan], sizeof(int32_t)*(s->blocksize + s->nwrap));
+        tmp_ptr = av_realloc(s->decoded_base[chan], (s->blocksize + s->nwrap) *
+                             sizeof(s->decoded_base[0][0]));
+        if (!tmp_ptr)
+            return AVERROR(ENOMEM);
+        s->decoded_base[chan] = tmp_ptr;
         for (i=0; i<s->nwrap; i++)
-            s->decoded[chan][i] = 0;
-        s->decoded[chan] += s->nwrap;
+            s->decoded_base[chan][i] = 0;
+        s->decoded[chan] = s->decoded_base[chan] + s->nwrap;
     }
     return 0;
 }
@@ -275,8 +285,15 @@ static int shorten_decode_frame(AVCodecContext *avctx,
     int i, input_buf_size = 0;
     int16_t *samples = data;
     if(s->max_framesize == 0){
+        void *tmp_ptr;
         s->max_framesize= 1024; // should hopefully be enough for the first header
-        s->bitstream= av_fast_realloc(s->bitstream, &s->allocated_bitstream_size, s->max_framesize);
+        tmp_ptr = av_fast_realloc(s->bitstream, &s->allocated_bitstream_size,
+                                  s->max_framesize);
+        if (!tmp_ptr) {
+            av_log(avctx, AV_LOG_ERROR, "error allocating bitstream buffer\n");
+            return AVERROR(ENOMEM);
+        }
+        s->bitstream = tmp_ptr;
     }
 
     if(1 && s->max_framesize){//FIXME truncated
@@ -514,8 +531,8 @@ static av_cold int shorten_decode_close(AVCodecContext *avctx)
     int i;
 
     for (i = 0; i < s->channels; i++) {
-        s->decoded[i] -= s->nwrap;
-        av_freep(&s->decoded[i]);
+        s->decoded[i] = NULL;
+        av_freep(&s->decoded_base[i]);
         av_freep(&s->offset[i]);
     }
     av_freep(&s->bitstream);
diff --git a/libavcodec/tiffenc.c b/libavcodec/tiffenc.c
index 1bc3c82c7e..4d6172fc46 100644
--- a/libavcodec/tiffenc.c
+++ b/libavcodec/tiffenc.c
@@ -304,6 +304,10 @@ static int encode_frame(AVCodecContext * avctx, unsigned char *buf,
 
     strip_sizes = av_mallocz(sizeof(*strip_sizes) * strips);
     strip_offsets = av_mallocz(sizeof(*strip_offsets) * strips);
+    if (!strip_sizes || !strip_offsets) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
 
     bytes_per_row = (((s->width - 1)/s->subsampling[0] + 1) * s->bpp
                     * s->subsampling[0] * s->subsampling[1] + 7) >> 3;
@@ -311,6 +315,7 @@ static int encode_frame(AVCodecContext * avctx, unsigned char *buf,
         yuv_line = av_malloc(bytes_per_row);
         if (yuv_line == NULL){
             av_log(s->avctx, AV_LOG_ERROR, "Not enough memory\n");
+            ret = AVERROR(ENOMEM);
             goto fail;
         }
     }
@@ -323,6 +328,10 @@ static int encode_frame(AVCodecContext * avctx, unsigned char *buf,
 
         zlen = bytes_per_row * s->rps;
         zbuf = av_malloc(zlen);
+        if (!zbuf) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
         strip_offsets[0] = ptr - buf;
         zn = 0;
         for (j = 0; j < s->rps; j++) {
@@ -347,8 +356,13 @@ static int encode_frame(AVCodecContext * avctx, unsigned char *buf,
     } else
 #endif
     {
-        if(s->compr == TIFF_LZW)
+        if (s->compr == TIFF_LZW) {
             s->lzws = av_malloc(ff_lzw_encode_state_size);
+            if (!s->lzws) {
+                ret = AVERROR(ENOMEM);
+                goto fail;
+            }
+        }
         for (i = 0; i < s->height; i++) {
             if (strip_sizes[i / s->rps] == 0) {
                 if(s->compr == TIFF_LZW){
diff --git a/libavcodec/vp56.c b/libavcodec/vp56.c
index 1ca8a402f0..f48dd15692 100644
--- a/libavcodec/vp56.c
+++ b/libavcodec/vp56.c
@@ -516,8 +516,14 @@ int vp56_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         s->modelp = &s->models[is_alpha];
 
         res = s->parse_header(s, buf, remaining_buf_size, &golden_frame);
-        if (!res)
-            return -1;
+        if (!res) {
+            int i;
+            for (i = 0; i < 4; i++) {
+                if (s->frames[i].data[0])
+                    avctx->release_buffer(avctx, &s->frames[i]);
+            }
+            return res;
+        }
 
         if (res == 2) {
             int i;
diff --git a/libavcodec/vp6.c b/libavcodec/vp6.c
index d9e9711cca..6d0d3b2d6e 100644
--- a/libavcodec/vp6.c
+++ b/libavcodec/vp6.c
@@ -61,8 +61,8 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
             return 0;
         s->filter_header = buf[1] & 0x06;
         if (buf[1] & 1) {
-            av_log(s->avctx, AV_LOG_ERROR, "interlacing not supported\n");
-            return 0;
+            av_log(s->avctx, AV_LOG_WARNING, "interlacing not supported\n");
+            return AVERROR_PATCHWELCOME;
         }
         if (separated_coeff || !s->filter_header) {
             coeff_offset = AV_RB16(buf+2) - 2;
diff --git a/libavfilter/formats.c b/libavfilter/formats.c
index c91f8b2cf8..39b3453e41 100644
--- a/libavfilter/formats.c
+++ b/libavfilter/formats.c
@@ -46,6 +46,9 @@ AVFilterFormats *avfilter_merge_formats(AVFilterFormats *a, AVFilterFormats *b)
     if (a == b)
         return a;
 
+    if (a == b)
+        return a;
+
     ret = av_mallocz(sizeof(AVFilterFormats));
 
     /* merge list of formats */
diff --git a/libavformat/avidec.c b/libavformat/avidec.c
index 78e5051e1e..46dffa11b8 100644
--- a/libavformat/avidec.c
+++ b/libavformat/avidec.c
@@ -780,13 +780,13 @@ resync:
             else
                 ast->frame_offset++;
         }
-        ast->remaining -= size;
+        ast->remaining -= err;
         if(!ast->remaining){
             avi->stream_index= -1;
             ast->packet_size= 0;
         }
 
-        return size;
+        return 0;
     }
 
     memset(d, -1, sizeof(int)*8);
diff --git a/libavformat/oggdec.c b/libavformat/oggdec.c
index a810b95dee..469f54dca6 100644
--- a/libavformat/oggdec.c
+++ b/libavformat/oggdec.c
@@ -66,8 +66,7 @@ ogg_save (AVFormatContext * s)
 
     for (i = 0; i < ogg->nstreams; i++){
         struct ogg_stream *os = ogg->streams + i;
-        os->buf = av_malloc (os->bufsize);
-        memset (os->buf, 0, os->bufsize);
+        os->buf = av_mallocz (os->bufsize + FF_INPUT_BUFFER_PADDING_SIZE);
         memcpy (os->buf, ost->streams[i].buf, os->bufpos);
     }
 
@@ -170,13 +169,18 @@ ogg_new_stream (AVFormatContext * s, uint32_t serial)
     AVStream *st;
     struct ogg_stream *os;
 
-    ogg->streams = av_realloc (ogg->streams,
-                               ogg->nstreams * sizeof (*ogg->streams));
+    os = av_realloc (ogg->streams, ogg->nstreams * sizeof (*ogg->streams));
+
+    if (!os)
+        return AVERROR(ENOMEM);
+
+    ogg->streams = os;
+
     memset (ogg->streams + idx, 0, sizeof (*ogg->streams));
     os = ogg->streams + idx;
     os->serial = serial;
     os->bufsize = DECODER_BUFFER_SIZE;
-    os->buf = av_malloc(os->bufsize);
+    os->buf = av_malloc(os->bufsize + FF_INPUT_BUFFER_PADDING_SIZE);
     os->header = -1;
 
     st = av_new_stream (s, idx);
@@ -192,7 +196,7 @@ static int
 ogg_new_buf(struct ogg *ogg, int idx)
 {
     struct ogg_stream *os = ogg->streams + idx;
-    uint8_t *nb = av_malloc(os->bufsize);
+    uint8_t *nb = av_malloc(os->bufsize + FF_INPUT_BUFFER_PADDING_SIZE);
     int size = os->bufpos - os->pstart;
     if(os->buf){
         memcpy(nb, os->buf + os->pstart, size);
@@ -289,7 +293,9 @@ ogg_read_page (AVFormatContext * s, int *str)
     }
 
     if (os->bufsize - os->bufpos < size){
-        uint8_t *nb = av_malloc (os->bufsize *= 2);
+        uint8_t *nb = av_malloc ((os->bufsize *= 2) + FF_INPUT_BUFFER_PADDING_SIZE);
+        if (!nb)
+            return AVERROR(ENOMEM);
         memcpy (nb, os->buf, os->bufpos);
         av_free (os->buf);
         os->buf = nb;
@@ -303,6 +309,7 @@ ogg_read_page (AVFormatContext * s, int *str)
     os->granule = gp;
     os->flags = flags;
 
+    memset(os->buf + os->bufpos, 0, FF_INPUT_BUFFER_PADDING_SIZE);
     if (str)
         *str = idx;
 
diff --git a/libavformat/utils.c b/libavformat/utils.c
index 223d567f75..271502327f 100644
--- a/libavformat/utils.c
+++ b/libavformat/utils.c
@@ -680,7 +680,10 @@ static void compute_frame_duration(int *pnum, int *pden, AVStream *st,
             *pnum = st->codec->time_base.num;
             *pden = st->codec->time_base.den;
             if (pc && pc->repeat_pict) {
-                *pnum = (*pnum) * (1 + pc->repeat_pict);
+                if (*pnum > INT_MAX / (1 + pc->repeat_pict))
+                    *pden /= 1 + pc->repeat_pict;
+                else
+                    *pnum *= 1 + pc->repeat_pict;
             }
         }
         break;
diff --git a/libavformat/yuv4mpeg.c b/libavformat/yuv4mpeg.c
index 3fd7927884..19e8be7636 100644
--- a/libavformat/yuv4mpeg.c
+++ b/libavformat/yuv4mpeg.c
@@ -152,6 +152,11 @@ static int yuv4_write_header(AVFormatContext *s)
     if (s->nb_streams != 1)
         return AVERROR(EIO);
 
+    if (s->streams[0]->codec->codec_id != CODEC_ID_RAWVIDEO) {
+        av_log(s, AV_LOG_ERROR, "ERROR: Only rawvideo supported.\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     if (s->streams[0]->codec->pix_fmt == PIX_FMT_YUV411P) {
         av_log(s, AV_LOG_ERROR, "Warning: generating rarely used 4:1:1 YUV stream, some mjpegtools might not work.\n");
     }
@@ -340,7 +345,7 @@ static int yuv4_read_packet(AVFormatContext *s, AVPacket *pkt)
 {
     int i;
     char header[MAX_FRAME_HEADER+1];
-    int packet_size, width, height;
+    int packet_size, width, height, ret;
     AVStream *st = s->streams[0];
     struct frame_attributes *s1 = s->priv_data;
 
@@ -351,18 +356,28 @@ static int yuv4_read_packet(AVFormatContext *s, AVPacket *pkt)
             break;
         }
     }
-    if (i == MAX_FRAME_HEADER) return -1;
-    if (strncmp(header, Y4M_FRAME_MAGIC, strlen(Y4M_FRAME_MAGIC))) return -1;
+    if (s->pb->error)
+        return s->pb->error;
+    else if (s->pb->eof_reached)
+        return AVERROR_EOF;
+    else if (i == MAX_FRAME_HEADER)
+        return AVERROR_INVALIDDATA;
+
+    if (strncmp(header, Y4M_FRAME_MAGIC, strlen(Y4M_FRAME_MAGIC)))
+        return AVERROR_INVALIDDATA;
 
     width = st->codec->width;
     height = st->codec->height;
 
     packet_size = avpicture_get_size(st->codec->pix_fmt, width, height);
     if (packet_size < 0)
-        return -1;
+        return packet_size;
 
-    if (av_get_packet(s->pb, pkt, packet_size) != packet_size)
-        return AVERROR(EIO);
+    ret = av_get_packet(s->pb, pkt, packet_size);
+    if (ret < 0)
+        return ret;
+    else if (ret != packet_size)
+        return s->pb->eof_reached ? AVERROR_EOF : AVERROR(EIO);
 
     if (s->streams[0]->codec->coded_frame) {
         s->streams[0]->codec->coded_frame->interlaced_frame = s1->interlaced_frame;