Merge branch 'release/0.8' into release/0.7

* release/0.8:
  shorten: Fix invalid free()
  j2kdec: Fix crash in get_qcx
  j2kdec: Check curtileno for validity
  atrac3: Fix crash in tonal component decoding. Fixes Ticket780 Bug Found by: cosminamironesei
  h264: check chroma_format_idc range. Fixes Ticket758 Bug found by: Diana Elena Muscalu
  aacsbr: Fix memory corruption. Fixes Ticket760 and Ticket761 Bug Found by: Diana Elena Muscalu
  j2kdec: Fix integer overflow leading to a segfault Fixes Ticket776 Bug found by: Diana Elena Muscalu
  ws_snd1: Fix wrong samples count and crash.
  lavfi: add missing check in avfilter_filter_samples()
  Update Changelog for 0.7.4 release
  Update RELEASE file for 0.7.4
  swscale: fix crash in fast_bilinear code when compiled with -mred-zone.
  vorbis: An additional defense in the Vorbis codec.
  vorbisdec: Fix decoding bug with channel handling

Merged-by: Michael Niedermayer <michaelni@gmx.at>

8 files changed, 56 insertions, 22 deletions

diff --git a/libavcodec/aacsbr.c b/libavcodec/aacsbr.c
index 10b8daf280..866482aac3 100644
--- a/libavcodec/aacsbr.c
+++ b/libavcodec/aacsbr.c
@@ -1185,7 +1185,7 @@ static void sbr_qmf_synthesis(DSPContext *dsp, FFTContext *mdct,
     const float *sbr_qmf_window = div ? sbr_qmf_window_ds : sbr_qmf_window_us;
     float *v;
     for (i = 0; i < 32; i++) {
-        if (*v_off == 0) {
+        if (*v_off < 128 >> div) {
             int saved_samples = (1280 - 128) >> div;
             memcpy(&v0[SBR_SYNTHESIS_BUF_SIZE - saved_samples], v0, saved_samples * sizeof(float));
             *v_off = SBR_SYNTHESIS_BUF_SIZE - saved_samples - (128 >> div);
diff --git a/libavcodec/atrac3.c b/libavcodec/atrac3.c
index 20ab75dfd7..f16630450d 100644
--- a/libavcodec/atrac3.c
+++ b/libavcodec/atrac3.c
@@ -395,6 +395,8 @@ static int decodeTonalComponents (GetBitContext *gb, tonal_component *pComponent
 
             for (k=0; k<coded_components; k++) {
                 sfIndx = get_bits(gb,6);
+                if(component_count>=64)
+                    return AVERROR_INVALIDDATA;
                 pComponent[component_count].pos = j * 64 + (get_bits(gb,6));
                 max_coded_values = 1024 - pComponent[component_count].pos;
                 coded_values = coded_values_per_component + 1;
diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c
index 89e2502e0d..680db1e5a6 100644
--- a/libavcodec/h264_ps.c
+++ b/libavcodec/h264_ps.c
@@ -342,6 +342,10 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){
 
     if(sps->profile_idc >= 100){ //high profile
         sps->chroma_format_idc= get_ue_golomb_31(&s->gb);
+        if (sps->chroma_format_idc > 3U) {
+            av_log(h->s.avctx, AV_LOG_ERROR, "chroma_format_idc %d is illegal\n", sps->chroma_format_idc);
+            goto fail;
+        }
         if(sps->chroma_format_idc == 3)
             sps->residual_color_transform_flag = get_bits1(&s->gb);
         sps->bit_depth_luma   = get_ue_golomb(&s->gb) + 8;
diff --git a/libavcodec/j2k_dwt.c b/libavcodec/j2k_dwt.c
index ab7a1ab757..48aa33735e 100644
--- a/libavcodec/j2k_dwt.c
+++ b/libavcodec/j2k_dwt.c
@@ -321,7 +321,7 @@ int ff_j2k_dwt_init(DWTContext *s, uint16_t border[2][2], int decomp_levels, int
     int i, j, lev = decomp_levels, maxlen,
         b[2][2];
 
-    if (decomp_levels >= FF_DWT_MAX_DECLVLS)
+    if ((unsigned)decomp_levels >= FF_DWT_MAX_DECLVLS)
         return AVERROR_INVALIDDATA;
     s->ndeclevels = decomp_levels;
     s->type = type;
diff --git a/libavcodec/j2kdec.c b/libavcodec/j2kdec.c
index 96b4f64098..3315a835d7 100644
--- a/libavcodec/j2kdec.c
+++ b/libavcodec/j2kdec.c
@@ -359,7 +359,7 @@ static int get_qcx(J2kDecoderContext *s, int n, J2kQuantStyle *q)
 
     if (q->quantsty == J2K_QSTY_NONE){
         n -= 3;
-        if (s->buf_end - s->buf < n)
+        if (s->buf_end - s->buf < n || 32*3 < n)
             return AVERROR(EINVAL);
         for (i = 0; i < n; i++)
             q->expn[i] = bytestream_get_byte(&s->buf) >> 3;
@@ -376,7 +376,7 @@ static int get_qcx(J2kDecoderContext *s, int n, J2kQuantStyle *q)
         }
     } else{
         n = (n - 3) >> 1;
-        if (s->buf_end - s->buf < n)
+        if (s->buf_end - s->buf < n || 32*3 < n)
             return AVERROR(EINVAL);
         for (i = 0; i < n; i++){
             x = bytestream_get_be16(&s->buf);
@@ -421,6 +421,10 @@ static uint8_t get_sot(J2kDecoderContext *s)
         return AVERROR(EINVAL);
 
     s->curtileno = bytestream_get_be16(&s->buf); ///< Isot
+    if((unsigned)s->curtileno >= s->numXtiles * s->numYtiles){
+        s->curtileno=0;
+        return AVERROR(EINVAL);
+    }
 
     s->buf += 4; ///< Psot (ignored)
 
diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c
index 0b9d420d86..621281fc75 100644
--- a/libavcodec/shorten.c
+++ b/libavcodec/shorten.c
@@ -81,6 +81,7 @@ typedef struct ShortenContext {
     int channels;
 
     int32_t *decoded[MAX_CHANNELS];
+    int32_t *decoded_base[MAX_CHANNELS];
     int32_t *offset[MAX_CHANNELS];
     int *coeffs;
     uint8_t *bitstream;
@@ -130,13 +131,13 @@ static int allocate_buffers(ShortenContext *s)
             return AVERROR(ENOMEM);
         s->offset[chan] = tmp_ptr;
 
-        tmp_ptr = av_realloc(s->decoded[chan], sizeof(int32_t)*(s->blocksize + s->nwrap));
+        tmp_ptr = av_realloc(s->decoded_base[chan], sizeof(int32_t)*(s->blocksize + s->nwrap));
         if (!tmp_ptr)
             return AVERROR(ENOMEM);
-        s->decoded[chan] = tmp_ptr;
+        s->decoded_base[chan] = tmp_ptr;
         for (i=0; i<s->nwrap; i++)
-            s->decoded[chan][i] = 0;
-        s->decoded[chan] += s->nwrap;
+            s->decoded_base[chan][i] = 0;
+        s->decoded[chan] = s->decoded_base[chan] + s->nwrap;
     }
 
     coeffs = av_realloc(s->coeffs, s->nwrap * sizeof(*s->coeffs));
@@ -548,8 +549,8 @@ static av_cold int shorten_decode_close(AVCodecContext *avctx)
     int i;
 
     for (i = 0; i < s->channels; i++) {
-        s->decoded[i] -= s->nwrap;
-        av_freep(&s->decoded[i]);
+        s->decoded[i] = NULL;
+        av_freep(&s->decoded_base[i]);
         av_freep(&s->offset[i]);
     }
     av_freep(&s->bitstream);
diff --git a/libavcodec/vorbisdec.c b/libavcodec/vorbisdec.c
index 8f16d3a5a5..ddb9bd6d5d 100644
--- a/libavcodec/vorbisdec.c
+++ b/libavcodec/vorbisdec.c
@@ -660,7 +660,7 @@ static int vorbis_parse_setup_hdr_residues(vorbis_context *vc)
         res_setup->partition_size = get_bits(gb, 24) + 1;
         /* Validations to prevent a buffer overflow later. */
         if (res_setup->begin>res_setup->end ||
-            res_setup->end > vc->avccontext->channels * vc->blocksize[1] / 2 ||
+            res_setup->end > (res_setup->type == 2 ? vc->avccontext->channels : 1) * vc->blocksize[1] / 2 ||
             (res_setup->end-res_setup->begin) / res_setup->partition_size > V_MAX_PARTITIONS) {
             av_log(vc->avccontext, AV_LOG_ERROR,
                    "partition out of bounds: type, begin, end, size, blocksize: %"PRIu16", %"PRIu32", %"PRIu32", %u, %"PRIu32"\n",
@@ -1269,6 +1269,7 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc,
                                                            uint8_t *do_not_decode,
                                                            float *vec,
                                                            unsigned vlen,
+                                                           unsigned ch_left,
                                                            int vr_type)
 {
     GetBitContext *gb = &vc->gb;
@@ -1276,6 +1277,7 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc,
     unsigned ptns_to_read = vr->ptns_to_read;
     uint8_t *classifs = vr->classifs;
     unsigned pass, ch_used, i, j, k, l;
+    unsigned max_output = (ch - 1) * vlen;
 
     if (vr_type == 2) {
         for (j = 1; j < ch; ++j)
@@ -1283,8 +1285,15 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc,
         if (do_not_decode[0])
             return 0;
         ch_used = 1;
+        max_output += vr->end / ch;
     } else {
         ch_used = ch;
+        max_output += vr->end;
+    }
+
+    if (max_output > ch_left * vlen) {
+        av_log(vc->avccontext, AV_LOG_ERROR, "Insufficient output buffer\n");
+        return -1;
     }
 
     av_dlog(NULL, " residue type 0/1/2 decode begin, ch: %d  cpc %d  \n", ch, c_p_c);
@@ -1411,14 +1420,15 @@ static av_always_inline int vorbis_residue_decode_internal(vorbis_context *vc,
 static inline int vorbis_residue_decode(vorbis_context *vc, vorbis_residue *vr,
                                         unsigned ch,
                                         uint8_t *do_not_decode,
-                                        float *vec, unsigned vlen)
+                                        float *vec, unsigned vlen,
+                                        unsigned ch_left)
 {
     if (vr->type == 2)
-        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, 2);
+        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, ch_left, 2);
     else if (vr->type == 1)
-        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, 1);
+        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, ch_left, 1);
     else if (vr->type == 0)
-        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, 0);
+        return vorbis_residue_decode_internal(vc, vr, ch, do_not_decode, vec, vlen, ch_left, 0);
     else {
         av_log(vc->avccontext, AV_LOG_ERROR, " Invalid residue type while residue decode?! \n");
         return -1;
@@ -1466,6 +1476,8 @@ static int vorbis_parse_audio_packet(vorbis_context *vc)
     uint8_t res_chan[255];
     unsigned res_num = 0;
     int retlen  = 0;
+    unsigned ch_left = vc->audio_channels;
+    unsigned vlen;
 
     if (get_bits1(gb)) {
         av_log(vc->avccontext, AV_LOG_ERROR, "Not a Vorbis I audio packet.\n");
@@ -1485,11 +1497,12 @@ static int vorbis_parse_audio_packet(vorbis_context *vc)
 
     blockflag = vc->modes[mode_number].blockflag;
     blocksize = vc->blocksize[blockflag];
+    vlen = blocksize / 2;
     if (blockflag)
         skip_bits(gb, 2); // previous_window, next_window
 
-    memset(ch_res_ptr,   0, sizeof(float) * vc->audio_channels * blocksize / 2); //FIXME can this be removed ?
-    memset(ch_floor_ptr, 0, sizeof(float) * vc->audio_channels * blocksize / 2); //FIXME can this be removed ?
+    memset(ch_res_ptr,   0, sizeof(float) * vc->audio_channels * vlen); //FIXME can this be removed ?
+    memset(ch_floor_ptr, 0, sizeof(float) * vc->audio_channels * vlen); //FIXME can this be removed ?
 
 // Decode floor
 
@@ -1509,7 +1522,7 @@ static int vorbis_parse_audio_packet(vorbis_context *vc)
             return -1;
         }
         no_residue[i] = ret;
-        ch_floor_ptr += blocksize / 2;
+        ch_floor_ptr += vlen;
     }
 
 // Nonzero vector propagate
@@ -1526,6 +1539,7 @@ static int vorbis_parse_audio_packet(vorbis_context *vc)
     for (i = 0; i < mapping->submaps; ++i) {
         vorbis_residue *residue;
         unsigned ch = 0;
+        int ret;
 
         for (j = 0; j < vc->audio_channels; ++j) {
             if ((mapping->submaps == 1) || (i == mapping->mux[j])) {
@@ -1540,9 +1554,18 @@ static int vorbis_parse_audio_packet(vorbis_context *vc)
             }
         }
         residue = &vc->residues[mapping->submap_residue[i]];
-        vorbis_residue_decode(vc, residue, ch, do_not_decode, ch_res_ptr, blocksize/2);
+        if (ch_left < ch) {
+            av_log(vc->avccontext, AV_LOG_ERROR, "Too many channels in vorbis_floor_decode.\n");
+            return -1;
+        }
+        if (ch) {
+            ret = vorbis_residue_decode(vc, residue, ch, do_not_decode, ch_res_ptr, vlen, ch_left);
+            if (ret < 0)
+                return ret;
+        }
 
-        ch_res_ptr += ch * blocksize / 2;
+        ch_res_ptr += ch * vlen;
+        ch_left -= ch;
     }
 
 // Inverse coupling
diff --git a/libavcodec/ws-snd1.c b/libavcodec/ws-snd1.c
index f92c3531e0..c28d1a8d6a 100644
--- a/libavcodec/ws-snd1.c
+++ b/libavcodec/ws-snd1.c
@@ -100,8 +100,8 @@ static int ws_snd_decode_frame(AVCodecContext *avctx,
 
         /* make sure we don't write more than out_size samples */
         switch (code) {
-        case 0:  smp = 4;                              break;
-        case 1:  smp = 2;                              break;
+        case 0:  smp = 4*(count+1);                    break;
+        case 1:  smp = 2*(count+1);                    break;
         case 2:  smp = (count & 0x20) ? 1 : count + 1; break;
         default: smp = count + 1;                      break;
         }