Merge commit '65830277d2d2ee3658e1f070a61044fff261ed3e' into release/1.1

* commit '65830277d2d2ee3658e1f070a61044fff261ed3e':
  prores: Add a codepath for decoding errors
  nut: Fix unchecked allocations
  avi: directly resync on DV in AVI read failure
  mov: Don't allocate arrays with av_malloc that will be realloced
  shorten: Extend fixed_coeffs to properly support pred_order 0
  Prepare for 9.11 RELEASE
  avi: properly fail if the dv demuxer is missing
  prores: Reject negative run and level values
  audio_mix: fix channel order in mix_1_to_2_fltp_flt_c
  indeo4: Check the inherited quant_mat

Conflicts:
	RELEASE
	libavcodec/indeo4.c
	libavcodec/shorten.c
	libavformat/nut.c
	libavformat/nutdec.c
	libavformat/nutenc.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

10 files changed, 89 insertions, 48 deletions

diff --git a/libavcodec/indeo4.c b/libavcodec/indeo4.c
index 8a2bcb3aa3..d7acdf75d0 100644
--- a/libavcodec/indeo4.c
+++ b/libavcodec/indeo4.c
@@ -399,6 +399,10 @@ static int decode_band_hdr(IVI45DecContext *ctx, IVIBandDesc *band,
                        "inherited\n");
                 return AVERROR_INVALIDDATA;
             }
+            if (band->quant_mat < 0) {
+                av_log(avctx, AV_LOG_ERROR, "Invalid quant_mat inherited\n");
+                return AVERROR_INVALIDDATA;
+            }
         }
         if (quant_index_to_tab[band->quant_mat] > 4 && band->blk_size == 4) {
             av_log(avctx, AV_LOG_ERROR, "Invalid quant matrix for 4x4 block encountered!\n");
diff --git a/libavcodec/proresdec_lgpl.c b/libavcodec/proresdec_lgpl.c
index a25ecca3c9..9308e74669 100644
--- a/libavcodec/proresdec_lgpl.c
+++ b/libavcodec/proresdec_lgpl.c
@@ -368,7 +368,7 @@ static inline void decode_dc_coeffs(GetBitContext *gb, DCTELEM *out,
 /**
  * Decode AC coefficients for all blocks in a slice.
  */
-static inline void decode_ac_coeffs(GetBitContext *gb, DCTELEM *out,
+static inline int decode_ac_coeffs(GetBitContext *gb, DCTELEM *out,
                                     int blocks_per_slice,
                                     int plane_size_factor,
                                     const uint8_t *scan)
@@ -389,15 +389,19 @@ static inline void decode_ac_coeffs(GetBitContext *gb, DCTELEM *out,
 
         bits_left = get_bits_left(gb);
         if (bits_left <= 0 || (bits_left <= 8 && !show_bits(gb, bits_left)))
-            return;
+            return AVERROR_INVALIDDATA;
 
         run = decode_vlc_codeword(gb, ff_prores_ac_codebook[run_cb_index]);
+        if (run < 0)
+            return AVERROR_INVALIDDATA;
 
         bits_left = get_bits_left(gb);
         if (bits_left <= 0 || (bits_left <= 8 && !show_bits(gb, bits_left)))
-            return;
+            return AVERROR_INVALIDDATA;
 
         level = decode_vlc_codeword(gb, ff_prores_ac_codebook[lev_cb_index]) + 1;
+        if (level < 0)
+            return AVERROR_INVALIDDATA;
 
         pos += run + 1;
         if (pos >= max_coeffs)
@@ -407,22 +411,24 @@ static inline void decode_ac_coeffs(GetBitContext *gb, DCTELEM *out,
         out[((pos & block_mask) << 6) + scan[pos >> plane_size_factor]] =
             (level ^ sign) - sign;
     }
+
+    return 0;
 }
 
 
 /**
  * Decode a slice plane (luma or chroma).
  */
-static void decode_slice_plane(ProresContext *ctx, ProresThreadData *td,
-                               const uint8_t *buf,
-                               int data_size, uint16_t *out_ptr,
-                               int linesize, int mbs_per_slice,
-                               int blocks_per_mb, int plane_size_factor,
-                               const int16_t *qmat, int is_chroma)
+static int decode_slice_plane(ProresContext *ctx, ProresThreadData *td,
+                              const uint8_t *buf,
+                              int data_size, uint16_t *out_ptr,
+                              int linesize, int mbs_per_slice,
+                              int blocks_per_mb, int plane_size_factor,
+                              const int16_t *qmat, int is_chroma)
 {
     GetBitContext gb;
     DCTELEM *block_ptr;
-    int mb_num, blocks_per_slice;
+    int mb_num, blocks_per_slice, ret;
 
     blocks_per_slice = mbs_per_slice * blocks_per_mb;
 
@@ -432,8 +438,10 @@ static void decode_slice_plane(ProresContext *ctx, ProresThreadData *td,
 
     decode_dc_coeffs(&gb, td->blocks, blocks_per_slice);
 
-    decode_ac_coeffs(&gb, td->blocks, blocks_per_slice,
-                     plane_size_factor, ctx->scantable.permutated);
+    ret = decode_ac_coeffs(&gb, td->blocks, blocks_per_slice,
+                           plane_size_factor, ctx->scantable.permutated);
+    if (ret < 0)
+        return ret;
 
     /* inverse quantization, inverse transform and output */
     block_ptr = td->blocks;
@@ -467,6 +475,7 @@ static void decode_slice_plane(ProresContext *ctx, ProresThreadData *td,
             }
         }
     }
+    return 0;
 }
 
 
@@ -485,6 +494,7 @@ static int decode_slice(AVCodecContext *avctx, void *tdata)
     int i, sf, slice_width_factor;
     int slice_data_size, hdr_size, y_data_size, u_data_size, v_data_size;
     int y_linesize, u_linesize, v_linesize;
+    int ret;
 
     buf             = ctx->slice_data[slice_num].index;
     slice_data_size = ctx->slice_data[slice_num + 1].index - buf;
@@ -541,28 +551,34 @@ static int decode_slice(AVCodecContext *avctx, void *tdata)
     }
 
     /* decode luma plane */
-    decode_slice_plane(ctx, td, buf + hdr_size, y_data_size,
-                       (uint16_t*) (y_data + (mb_y_pos << 4) * y_linesize +
-                                    (mb_x_pos << 5)), y_linesize,
-                       mbs_per_slice, 4, slice_width_factor + 2,
-                       td->qmat_luma_scaled, 0);
+    ret = decode_slice_plane(ctx, td, buf + hdr_size, y_data_size,
+                             (uint16_t*) (y_data + (mb_y_pos << 4) * y_linesize +
+                                          (mb_x_pos << 5)), y_linesize,
+                             mbs_per_slice, 4, slice_width_factor + 2,
+                             td->qmat_luma_scaled, 0);
+    if (ret < 0)
+        return ret;
 
     /* decode U chroma plane */
-    decode_slice_plane(ctx, td, buf + hdr_size + y_data_size, u_data_size,
-                       (uint16_t*) (u_data + (mb_y_pos << 4) * u_linesize +
-                                    (mb_x_pos << ctx->mb_chroma_factor)),
-                       u_linesize, mbs_per_slice, ctx->num_chroma_blocks,
-                       slice_width_factor + ctx->chroma_factor - 1,
-                       td->qmat_chroma_scaled, 1);
+    ret = decode_slice_plane(ctx, td, buf + hdr_size + y_data_size, u_data_size,
+                             (uint16_t*) (u_data + (mb_y_pos << 4) * u_linesize +
+                                          (mb_x_pos << ctx->mb_chroma_factor)),
+                             u_linesize, mbs_per_slice, ctx->num_chroma_blocks,
+                             slice_width_factor + ctx->chroma_factor - 1,
+                             td->qmat_chroma_scaled, 1);
+    if (ret < 0)
+        return ret;
 
     /* decode V chroma plane */
-    decode_slice_plane(ctx, td, buf + hdr_size + y_data_size + u_data_size,
-                       v_data_size,
-                       (uint16_t*) (v_data + (mb_y_pos << 4) * v_linesize +
-                                    (mb_x_pos << ctx->mb_chroma_factor)),
-                       v_linesize, mbs_per_slice, ctx->num_chroma_blocks,
-                       slice_width_factor + ctx->chroma_factor - 1,
-                       td->qmat_chroma_scaled, 1);
+    ret = decode_slice_plane(ctx, td, buf + hdr_size + y_data_size + u_data_size,
+                             v_data_size,
+                             (uint16_t*) (v_data + (mb_y_pos << 4) * v_linesize +
+                                          (mb_x_pos << ctx->mb_chroma_factor)),
+                             v_linesize, mbs_per_slice, ctx->num_chroma_blocks,
+                             slice_width_factor + ctx->chroma_factor - 1,
+                             td->qmat_chroma_scaled, 1);
+    if (ret < 0)
+        return ret;
 
     return 0;
 }
diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c
index 1ba93b95da..eef322a2a2 100644
--- a/libavcodec/shorten.c
+++ b/libavcodec/shorten.c
@@ -273,7 +273,8 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header,
     return 0;
 }
 
-static const int fixed_coeffs[3][3] = {
+static const int fixed_coeffs[][3] = {
+    { 0,  0,  0 },
     { 1,  0,  0 },
     { 2, -1,  0 },
     { 3, -3,  1 }
@@ -302,7 +303,12 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel,
     } else {
         /* fixed LPC coeffs */
         pred_order = command;
-        coeffs     = fixed_coeffs[pred_order - 1];
+        if (pred_order >= FF_ARRAY_ELEMS(fixed_coeffs)) {
+            av_log(s->avctx, AV_LOG_ERROR, "invalid pred_order %d\n",
+                   pred_order);
+            return AVERROR_INVALIDDATA;
+        }
+        coeffs     = fixed_coeffs[pred_order];
         qshift     = 0;
     }
 
diff --git a/libavformat/avidec.c b/libavformat/avidec.c
index f33d3a7108..d48829ed99 100644
--- a/libavformat/avidec.c
+++ b/libavformat/avidec.c
@@ -490,7 +490,8 @@ static int avi_read_header(AVFormatContext *s)
                     avi->dv_demux = avpriv_dv_init_demux(s);
                     if (!avi->dv_demux)
                         goto fail;
-                }
+                } else
+                    goto fail;
                 s->streams[0]->priv_data = ast;
                 avio_skip(pb, 3 * 4);
                 ast->scale = avio_rl32(pb);
@@ -1067,6 +1068,8 @@ static int avi_read_packet(AVFormatContext *s, AVPacket *pkt)
         int size = avpriv_dv_get_packet(avi->dv_demux, pkt);
         if (size >= 0)
             return size;
+        else
+            goto resync;
     }
 
     if(avi->non_interleaved){
diff --git a/libavformat/mov.c b/libavformat/mov.c
index 905647edcc..2e069ead13 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -2524,7 +2524,7 @@ static int mov_read_trun(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     if (!sc->ctts_count && sc->sample_count)
     {
         /* Complement ctts table if moov atom doesn't have ctts atom. */
-        ctts_data = av_malloc(sizeof(*sc->ctts_data));
+        ctts_data = av_realloc(NULL, sizeof(*sc->ctts_data));
         if (!ctts_data)
             return AVERROR(ENOMEM);
         sc->ctts_data = ctts_data;
diff --git a/libavformat/nut.c b/libavformat/nut.c
index 62a650dbed..699e90a2b8 100644
--- a/libavformat/nut.c
+++ b/libavformat/nut.c
@@ -210,10 +210,16 @@ int ff_nut_sp_pts_cmp(const Syncpoint *a, const Syncpoint *b){
     return ((a->ts - b->ts) >> 32) - ((b->ts - a->ts) >> 32);
 }
 
-void ff_nut_add_sp(NUTContext *nut, int64_t pos, int64_t back_ptr, int64_t ts){
+int ff_nut_add_sp(NUTContext *nut, int64_t pos, int64_t back_ptr, int64_t ts){
     Syncpoint *sp= av_mallocz(sizeof(Syncpoint));
     struct AVTreeNode *node = av_tree_node_alloc();
 
+    if (!sp || !node) {
+        av_freep(&sp);
+        av_freep(&node);
+        return AVERROR(ENOMEM);
+    }
+
     nut->sp_count++;
 
     sp->pos= pos;
@@ -224,6 +230,8 @@ void ff_nut_add_sp(NUTContext *nut, int64_t pos, int64_t back_ptr, int64_t ts){
         av_free(sp);
         av_free(node);
     }
+
+    return 0;
 }
 
 static int enu_free(void *opaque, void *elem)
diff --git a/libavformat/nut.h b/libavformat/nut.h
index ab31c27429..f38406eff8 100644
--- a/libavformat/nut.h
+++ b/libavformat/nut.h
@@ -123,7 +123,7 @@ void ff_nut_reset_ts(NUTContext *nut, AVRational time_base, int64_t val);
 int64_t ff_lsb2full(StreamContext *stream, int64_t lsb);
 int ff_nut_sp_pos_cmp(const Syncpoint *a, const Syncpoint *b);
 int ff_nut_sp_pts_cmp(const Syncpoint *a, const Syncpoint *b);
-void ff_nut_add_sp(NUTContext *nut, int64_t pos, int64_t back_ptr, int64_t ts);
+int ff_nut_add_sp(NUTContext *nut, int64_t pos, int64_t back_ptr, int64_t ts);
 void ff_nut_free_sp(NUTContext *nut);
 
 extern const Dispositions ff_nut_dispositions[];
diff --git a/libavformat/nutdec.c b/libavformat/nutdec.c
index f260a3e4fd..c44e7b1e16 100644
--- a/libavformat/nutdec.c
+++ b/libavformat/nutdec.c
@@ -546,6 +546,7 @@ static int decode_syncpoint(NUTContext *nut, int64_t *ts, int64_t *back_ptr)
     AVIOContext *bc    = s->pb;
     int64_t end;
     uint64_t tmp;
+    int ret;
 
     nut->last_syncpoint_pos = avio_tell(bc) - 8;
 
@@ -567,7 +568,9 @@ static int decode_syncpoint(NUTContext *nut, int64_t *ts, int64_t *back_ptr)
 
     *ts = tmp / nut->time_base_count *
           av_q2d(nut->time_base[tmp % nut->time_base_count]) * AV_TIME_BASE;
-    ff_nut_add_sp(nut, nut->last_syncpoint_pos, *back_ptr, *ts);
+
+    if ((ret = ff_nut_add_sp(nut, nut->last_syncpoint_pos, *back_ptr, *ts)) < 0)
+        return ret;
 
     return 0;
 }
diff --git a/libavformat/nutenc.c b/libavformat/nutenc.c
index 9b1ffaf935..7924530386 100644
--- a/libavformat/nutenc.c
+++ b/libavformat/nutenc.c
@@ -865,7 +865,8 @@ static int nut_write_packet(AVFormatContext *s, AVPacket *pkt)
         ff_put_v(dyn_bc, sp ? (nut->last_syncpoint_pos - sp->pos) >> 4 : 0);
         put_packet(nut, bc, dyn_bc, 1, SYNCPOINT_STARTCODE);
 
-        ff_nut_add_sp(nut, nut->last_syncpoint_pos, 0 /*unused*/, pkt->dts);
+        if ((ret = ff_nut_add_sp(nut, nut->last_syncpoint_pos, 0 /*unused*/, pkt->dts)) < 0)
+            return ret;
 
         if ((1ll<<60) % nut->sp_count == 0)
             for (i=0; i<s->nb_streams; i++) {
diff --git a/libavresample/audio_mix.c b/libavresample/audio_mix.c
index c0560288a5..f737a30fc8 100644
--- a/libavresample/audio_mix.c
+++ b/libavresample/audio_mix.c
@@ -195,23 +195,23 @@ static void mix_1_to_2_fltp_flt_c(float **samples, float **matrix, int len,
 
     while (len > 4) {
         v = *src++;
-        *dst0++ = v * m1;
-        *dst1++ = v * m0;
+        *dst0++ = v * m0;
+        *dst1++ = v * m1;
         v = *src++;
-        *dst0++ = v * m1;
-        *dst1++ = v * m0;
+        *dst0++ = v * m0;
+        *dst1++ = v * m1;
         v = *src++;
-        *dst0++ = v * m1;
-        *dst1++ = v * m0;
+        *dst0++ = v * m0;
+        *dst1++ = v * m1;
         v = *src++;
-        *dst0++ = v * m1;
-        *dst1++ = v * m0;
+        *dst0++ = v * m0;
+        *dst1++ = v * m1;
         len -= 4;
     }
     while (len > 0) {
         v = *src++;
-        *dst0++ = v * m1;
-        *dst1++ = v * m0;
+        *dst0++ = v * m0;
+        *dst1++ = v * m1;
         len--;
     }
 }