Merge remote-tracking branch 'qatar/release/0.5' into release/0.5

* qatar/release/0.5: (21 commits)
  vp6: properly fail on unsupported feature
  vp56: release frames on error
  shorten: Use separate pointers for the allocated memory for decoded samples.
  shorten: check for realloc failure
  h264: check context state before decoding slice data partitions
  oggdec: check memory allocation
  Fix uninitialized reads on malformed ogg files.
  lavf: avoid integer overflow in ff_compute_frame_duration()
  yuv4mpeg: reject unsupported codecs
  tiffenc: Check av_malloc() results.
  mpegaudiodec: fix short_start calculation
  h264: avoid stuck buffer pointer in decode_nal_units
  yuv4mpeg: return proper error codes.
  avidec: return 0, not packet size from read_packet().
  cavsdec: check for changing w/h.
  avidec: use actually read size instead of requested size
  bytestream: add a new set of bytestream functions with overread checking
  avsdec: Set dimensions instead of relying on the demuxer.
  lavfi: avfilter_merge_formats: handle case where inputs are same
  bmpdec: only initialize palette for pal8.
  ...

Merged-by: Michael Niedermayer <michaelni@gmx.at>

4 files changed, 41 insertions, 16 deletions

diff --git a/libavformat/avidec.c b/libavformat/avidec.c
index 78e5051e1e..46dffa11b8 100644
--- a/libavformat/avidec.c
+++ b/libavformat/avidec.c
@@ -780,13 +780,13 @@ resync:
             else
                 ast->frame_offset++;
         }
-        ast->remaining -= size;
+        ast->remaining -= err;
         if(!ast->remaining){
             avi->stream_index= -1;
             ast->packet_size= 0;
         }
 
-        return size;
+        return 0;
     }
 
     memset(d, -1, sizeof(int)*8);
diff --git a/libavformat/oggdec.c b/libavformat/oggdec.c
index a810b95dee..469f54dca6 100644
--- a/libavformat/oggdec.c
+++ b/libavformat/oggdec.c
@@ -66,8 +66,7 @@ ogg_save (AVFormatContext * s)
 
     for (i = 0; i < ogg->nstreams; i++){
         struct ogg_stream *os = ogg->streams + i;
-        os->buf = av_malloc (os->bufsize);
-        memset (os->buf, 0, os->bufsize);
+        os->buf = av_mallocz (os->bufsize + FF_INPUT_BUFFER_PADDING_SIZE);
         memcpy (os->buf, ost->streams[i].buf, os->bufpos);
     }
 
@@ -170,13 +169,18 @@ ogg_new_stream (AVFormatContext * s, uint32_t serial)
     AVStream *st;
     struct ogg_stream *os;
 
-    ogg->streams = av_realloc (ogg->streams,
-                               ogg->nstreams * sizeof (*ogg->streams));
+    os = av_realloc (ogg->streams, ogg->nstreams * sizeof (*ogg->streams));
+
+    if (!os)
+        return AVERROR(ENOMEM);
+
+    ogg->streams = os;
+
     memset (ogg->streams + idx, 0, sizeof (*ogg->streams));
     os = ogg->streams + idx;
     os->serial = serial;
     os->bufsize = DECODER_BUFFER_SIZE;
-    os->buf = av_malloc(os->bufsize);
+    os->buf = av_malloc(os->bufsize + FF_INPUT_BUFFER_PADDING_SIZE);
     os->header = -1;
 
     st = av_new_stream (s, idx);
@@ -192,7 +196,7 @@ static int
 ogg_new_buf(struct ogg *ogg, int idx)
 {
     struct ogg_stream *os = ogg->streams + idx;
-    uint8_t *nb = av_malloc(os->bufsize);
+    uint8_t *nb = av_malloc(os->bufsize + FF_INPUT_BUFFER_PADDING_SIZE);
     int size = os->bufpos - os->pstart;
     if(os->buf){
         memcpy(nb, os->buf + os->pstart, size);
@@ -289,7 +293,9 @@ ogg_read_page (AVFormatContext * s, int *str)
     }
 
     if (os->bufsize - os->bufpos < size){
-        uint8_t *nb = av_malloc (os->bufsize *= 2);
+        uint8_t *nb = av_malloc ((os->bufsize *= 2) + FF_INPUT_BUFFER_PADDING_SIZE);
+        if (!nb)
+            return AVERROR(ENOMEM);
         memcpy (nb, os->buf, os->bufpos);
         av_free (os->buf);
         os->buf = nb;
@@ -303,6 +309,7 @@ ogg_read_page (AVFormatContext * s, int *str)
     os->granule = gp;
     os->flags = flags;
 
+    memset(os->buf + os->bufpos, 0, FF_INPUT_BUFFER_PADDING_SIZE);
     if (str)
         *str = idx;
 
diff --git a/libavformat/utils.c b/libavformat/utils.c
index 223d567f75..271502327f 100644
--- a/libavformat/utils.c
+++ b/libavformat/utils.c
@@ -680,7 +680,10 @@ static void compute_frame_duration(int *pnum, int *pden, AVStream *st,
             *pnum = st->codec->time_base.num;
             *pden = st->codec->time_base.den;
             if (pc && pc->repeat_pict) {
-                *pnum = (*pnum) * (1 + pc->repeat_pict);
+                if (*pnum > INT_MAX / (1 + pc->repeat_pict))
+                    *pden /= 1 + pc->repeat_pict;
+                else
+                    *pnum *= 1 + pc->repeat_pict;
             }
         }
         break;
diff --git a/libavformat/yuv4mpeg.c b/libavformat/yuv4mpeg.c
index 3fd7927884..19e8be7636 100644
--- a/libavformat/yuv4mpeg.c
+++ b/libavformat/yuv4mpeg.c
@@ -152,6 +152,11 @@ static int yuv4_write_header(AVFormatContext *s)
     if (s->nb_streams != 1)
         return AVERROR(EIO);
 
+    if (s->streams[0]->codec->codec_id != CODEC_ID_RAWVIDEO) {
+        av_log(s, AV_LOG_ERROR, "ERROR: Only rawvideo supported.\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     if (s->streams[0]->codec->pix_fmt == PIX_FMT_YUV411P) {
         av_log(s, AV_LOG_ERROR, "Warning: generating rarely used 4:1:1 YUV stream, some mjpegtools might not work.\n");
     }
@@ -340,7 +345,7 @@ static int yuv4_read_packet(AVFormatContext *s, AVPacket *pkt)
 {
     int i;
     char header[MAX_FRAME_HEADER+1];
-    int packet_size, width, height;
+    int packet_size, width, height, ret;
     AVStream *st = s->streams[0];
     struct frame_attributes *s1 = s->priv_data;
 
@@ -351,18 +356,28 @@ static int yuv4_read_packet(AVFormatContext *s, AVPacket *pkt)
             break;
         }
     }
-    if (i == MAX_FRAME_HEADER) return -1;
-    if (strncmp(header, Y4M_FRAME_MAGIC, strlen(Y4M_FRAME_MAGIC))) return -1;
+    if (s->pb->error)
+        return s->pb->error;
+    else if (s->pb->eof_reached)
+        return AVERROR_EOF;
+    else if (i == MAX_FRAME_HEADER)
+        return AVERROR_INVALIDDATA;
+
+    if (strncmp(header, Y4M_FRAME_MAGIC, strlen(Y4M_FRAME_MAGIC)))
+        return AVERROR_INVALIDDATA;
 
     width = st->codec->width;
     height = st->codec->height;
 
     packet_size = avpicture_get_size(st->codec->pix_fmt, width, height);
     if (packet_size < 0)
-        return -1;
+        return packet_size;
 
-    if (av_get_packet(s->pb, pkt, packet_size) != packet_size)
-        return AVERROR(EIO);
+    ret = av_get_packet(s->pb, pkt, packet_size);
+    if (ret < 0)
+        return ret;
+    else if (ret != packet_size)
+        return s->pb->eof_reached ? AVERROR_EOF : AVERROR(EIO);
 
     if (s->streams[0]->codec->coded_frame) {
         s->streams[0]->codec->coded_frame->interlaced_frame = s1->interlaced_frame;