Merge remote-tracking branch 'qatar/release/0.5' into release/0.5

* qatar/release/0.5:
  Bump version number for 0.5.8 release.
  Release notes and changelog for 0.5.7
  vqavideo: return error if image size is not a multiple of block size
  motionpixels: Clip YUV values after applying a gradient.
  mjpegbdec: Fix overflow in SOS.
  atrac3: Fix crash in tonal component decoding.
  dv: Fix small stack overread related to CVE-2011-3929 and CVE-2011-3936.
  dv: Fix null pointer dereference due to ach=0
  dv: check stype
  nsvdec: Propagate errors
  nsvdec: Be more careful with av_malloc().
  nsvdec: Fix use of uninitialized streams.

Conflicts:
	libavcodec/atrac3.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

7 files changed, 75 insertions, 15 deletions

diff --git a/Changelog b/Changelog
index aaea7350c4..68ccc62067 100644
--- a/Changelog
+++ b/Changelog
@@ -2,6 +2,20 @@ Entries are sorted chronologically from oldest to youngest within each release,
 releases are sorted from youngest to oldest.
 
 
+version 0.5.8:
+
+- id3v2: fix skipping extended header in id3v2.4
+- nsvdec: Several bugfixes related to CVE-2011-3940
+- dv: check stype
+- dv: Fix null pointer dereference due to ach=0
+- dv: Fix small stack overread related to CVE-2011-3929 and CVE-2011-3936.
+- atrac3: Fix crash in tonal component decoding, fixes CVE-2012-0853
+- mjpegbdec: Fix overflow in SOS, fixes CVE-2011-3947
+- motionpixels: Clip YUV values after applying a gradient.
+- vqavideo: return error if image size is not a multiple of block size,
+  fixes CVE-2012-0947.
+
+
 version 0.5.7:
 - vorbis: An additional defense in the Vorbis codec. (CVE-2011-3895)
 - vorbisdec: Fix decoding bug with channel handling.
diff --git a/RELEASE b/RELEASE
index a49bbfc592..1d7aa2b768 100644
--- a/RELEASE
+++ b/RELEASE
@@ -197,3 +197,19 @@ demuxer (CVE-2011-3893 and CVE-2011-3895).
 Distributors and system integrators are encouraged
 to update and share their patches against this branch.  For a full list
 of changes please see the Changelog file.
+
+* 0.5.8 May 10, 2012
+
+General notes
+-------------
+
+This maintenance-only release that addresses a number a number of
+security issues that have been brought to our attention. Among other
+(rather minor) fixes, this release features fixes for the DV decoder
+(CVE-2011-3929 and CVE-2011-3936), nsvdec (CVE-2011-3940), Atrac3
+(CVE-2012-0853), mjpegdec (CVE-2011-3947) and the VQA video decoder
+(CVE-2012-0947).
+
+Distributors and system integrators are encouraged
+to update and share their patches against this branch.  For a full list
+of changes please see the Changelog file.
diff --git a/libavcodec/mjpegbdec.c b/libavcodec/mjpegbdec.c
index 62b29e0623..f19a87ff96 100644
--- a/libavcodec/mjpegbdec.c
+++ b/libavcodec/mjpegbdec.c
@@ -49,6 +49,9 @@ read_header:
     s->restart_count = 0;
     s->mjpb_skiptosod = 0;
 
+    if (buf_end - buf_ptr >= 1 << 28)
+        return AVERROR_INVALIDDATA;
+
     init_get_bits(&hgb, buf_ptr, /*buf_size*/(buf_end - buf_ptr)*8);
 
     skip_bits(&hgb, 32); /* reserved zeros */
@@ -99,8 +102,8 @@ read_header:
     av_log(avctx, AV_LOG_DEBUG, "sod offs: 0x%x\n", sod_offs);
     if (sos_offs)
     {
-//        init_get_bits(&s->gb, buf+sos_offs, (buf_end - (buf+sos_offs))*8);
-        init_get_bits(&s->gb, buf_ptr+sos_offs, field_size*8);
+        init_get_bits(&s->gb, buf_ptr + sos_offs,
+                      8 * FFMIN(field_size, buf_end - buf_ptr - sos_offs));
         s->mjpb_skiptosod = (sod_offs - sos_offs - show_bits(&s->gb, 16));
         s->start_code = SOS;
         ff_mjpeg_decode_sos(s);
diff --git a/libavcodec/motionpixels.c b/libavcodec/motionpixels.c
index 2839d0718c..7971b09761 100644
--- a/libavcodec/motionpixels.c
+++ b/libavcodec/motionpixels.c
@@ -239,10 +239,13 @@ static void mp_decode_line(MotionPixelsContext *mp, GetBitContext *gb, int y)
             p = mp_get_yuv_from_rgb(mp, x - 1, y);
         } else {
             p.y += mp_gradient(mp, 0, mp_get_vlc(mp, gb));
+            p.y = av_clip(p.y, 0, 31);
             if ((x & 3) == 0) {
                 if ((y & 3) == 0) {
                     p.v += mp_gradient(mp, 1, mp_get_vlc(mp, gb));
+                    p.v = av_clip(p.v, -32, 31);
                     p.u += mp_gradient(mp, 2, mp_get_vlc(mp, gb));
+                    p.u = av_clip(p.u, -32, 31);
                     mp->hpt[((y / 4) * mp->avctx->width + x) / 4] = p;
                 } else {
                     p.v = mp->hpt[((y / 4) * mp->avctx->width + x) / 4].v;
@@ -266,9 +269,12 @@ static void mp_decode_frame_helper(MotionPixelsContext *mp, GetBitContext *gb)
             p = mp_get_yuv_from_rgb(mp, 0, y);
         } else {
             p.y += mp_gradient(mp, 0, mp_get_vlc(mp, gb));
+            p.y = av_clip(p.y, 0, 31);
             if ((y & 3) == 0) {
                 p.v += mp_gradient(mp, 1, mp_get_vlc(mp, gb));
+                p.v = av_clip(p.v, -32, 31);
                 p.u += mp_gradient(mp, 2, mp_get_vlc(mp, gb));
+                p.u = av_clip(p.u, -32, 31);
             }
             mp->vpt[y] = p;
             mp_set_rgb_from_yuv(mp, 0, y, &p);
diff --git a/libavcodec/vqavideo.c b/libavcodec/vqavideo.c
index 00df736d3b..f34a63173b 100644
--- a/libavcodec/vqavideo.c
+++ b/libavcodec/vqavideo.c
@@ -163,6 +163,12 @@ static av_cold int vqa_decode_init(AVCodecContext *avctx)
         return -1;
     }
 
+    if (s->width  & (s->vector_width  - 1) ||
+        s->height & (s->vector_height - 1)) {
+        av_log(avctx, AV_LOG_ERROR, "Image size not multiple of block size\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     /* allocate codebooks */
     s->codebook_size = MAX_CODEBOOK_SIZE;
     s->codebook = av_malloc(s->codebook_size);
diff --git a/libavformat/dv.c b/libavformat/dv.c
index 820c3b5cec..256dcd4312 100644
--- a/libavformat/dv.c
+++ b/libavformat/dv.c
@@ -125,10 +125,14 @@ static int dv_extract_audio(uint8_t* frame, uint8_t* ppcm[4],
     /* We work with 720p frames split in half, thus even frames have
      * channels 0,1 and odd 2,3. */
     ipcm = (sys->height == 720 && !(frame[1] & 0x0C)) ? 2 : 0;
-    pcm  = ppcm[ipcm++];
 
     /* for each DIF channel */
     for (chan = 0; chan < sys->n_difchan; chan++) {
+        /* next stereo channel (50Mbps and 100Mbps only) */
+        pcm = ppcm[ipcm++];
+        if (!pcm)
+            break;
+
         /* for each DIF segment */
         for (i = 0; i < sys->difseg_size; i++) {
             frame += 6 * 80; /* skip DIF segment header */
@@ -176,11 +180,6 @@ static int dv_extract_audio(uint8_t* frame, uint8_t* ppcm[4],
                 frame += 16 * 80; /* 15 Video DIFs + 1 Audio DIF */
             }
         }
-
-        /* next stereo channel (50Mbps and 100Mbps only) */
-        pcm = ppcm[ipcm++];
-        if (!pcm)
-            break;
     }
 
     return size;
@@ -202,6 +201,12 @@ static int dv_extract_audio_info(DVDemuxContext* c, uint8_t* frame)
     stype = (as_pack[3] & 0x1f);      /* 0 - 2CH, 2 - 4CH, 3 - 8CH */
     quant =  as_pack[4] & 0x07;       /* 0 - 16bit linear, 1 - 12bit nonlinear */
 
+    if (stype > 3) {
+        av_log(c->fctx, AV_LOG_ERROR, "stype %d is invalid\n", stype);
+        c->ach = 0;
+        return 0;
+    }
+
     /* note: ach counts PAIRS of channels (i.e. stereo channels) */
     ach = ((int[4]){  1,  0,  2,  4})[stype];
     if (ach == 1 && quant && freq == 2)
@@ -335,7 +340,8 @@ int dv_produce_packet(DVDemuxContext *c, AVPacket *pkt,
        c->audio_pkt[i].pts  = c->abytes * 30000*8 / c->ast[i]->codec->bit_rate;
        ppcm[i] = c->audio_buf[i];
     }
-    dv_extract_audio(buf, ppcm, c->sys);
+    if (c->ach)
+        dv_extract_audio(buf, ppcm, c->sys);
     c->abytes += size;
 
     /* We work with 720p frames split in half, thus even frames have
diff --git a/libavformat/nsvdec.c b/libavformat/nsvdec.c
index 719337c898..25b3d6c4ae 100644
--- a/libavformat/nsvdec.c
+++ b/libavformat/nsvdec.c
@@ -317,7 +317,9 @@ static int nsv_parse_NSVf_header(AVFormatContext *s, AVFormatParameters *ap)
         char *token, *value;
         char quote;
 
-        p = strings = av_mallocz(strings_size + 1);
+        p = strings = av_mallocz((size_t)strings_size + 1);
+        if (!p)
+            return AVERROR(ENOMEM);
         endp = strings + strings_size;
         get_buffer(pb, strings, strings_size);
         while (p < endp) {
@@ -351,6 +353,8 @@ static int nsv_parse_NSVf_header(AVFormatContext *s, AVFormatParameters *ap)
         if((unsigned)table_entries >= UINT_MAX / sizeof(uint32_t))
             return -1;
         nsv->nsvf_index_data = av_malloc(table_entries * sizeof(uint32_t));
+        if (!nsv->nsvf_index_data)
+            return AVERROR(ENOMEM);
 #warning "FIXME: Byteswap buffer as needed"
         get_buffer(pb, (unsigned char *)nsv->nsvf_index_data, table_entries * sizeof(uint32_t));
     }
@@ -507,11 +511,16 @@ static int nsv_read_header(AVFormatContext *s, AVFormatParameters *ap)
     for (i = 0; i < NSV_MAX_RESYNC_TRIES; i++) {
         if (nsv_resync(s) < 0)
             return -1;
-        if (nsv->state == NSV_FOUND_NSVF)
+        if (nsv->state == NSV_FOUND_NSVF) {
             err = nsv_parse_NSVf_header(s, ap);
+            if (err < 0)
+                return err;
+        }
             /* we need the first NSVs also... */
         if (nsv->state == NSV_FOUND_NSVS) {
             err = nsv_parse_NSVs_header(s, ap);
+            if (err < 0)
+                return err;
             break; /* we just want the first one */
         }
     }
@@ -586,12 +595,12 @@ null_chunk_retry:
     }
 
     /* map back streams to v,a */
-    if (s->streams[0])
+    if (s->nb_streams > 0)
         st[s->streams[0]->id] = s->streams[0];
-    if (s->streams[1])
+    if (s->nb_streams > 1)
         st[s->streams[1]->id] = s->streams[1];
 
-    if (vsize/* && st[NSV_ST_VIDEO]*/) {
+    if (vsize && st[NSV_ST_VIDEO]) {
         nst = st[NSV_ST_VIDEO]->priv_data;
         pkt = &nsv->ahead[NSV_ST_VIDEO];
         av_get_packet(pb, pkt, vsize);
@@ -606,7 +615,7 @@ null_chunk_retry:
     if(st[NSV_ST_VIDEO])
         ((NSVStream*)st[NSV_ST_VIDEO]->priv_data)->frame_offset++;
 
-    if (asize/*st[NSV_ST_AUDIO]*/) {
+    if (asize && st[NSV_ST_AUDIO]) {
         nst = st[NSV_ST_AUDIO]->priv_data;
         pkt = &nsv->ahead[NSV_ST_AUDIO];
         /* read raw audio specific header on the first audio chunk... */