Merge commit 'b143844ea0f6246e0d5a938d743e2e8a98453bec' into release/0.8

* commit 'b143844ea0f6246e0d5a938d743e2e8a98453bec': (22 commits)
  aacdec: Fix an off-by-one overwrite when switching to LTP profile from MAIN.
  vp6: properly fail on unsupported feature
  h264: Fix parameters to ff_er_add_slice() call
  flacenc: ensure the order is within the min/max range in LPC order search
  yuv4mpeg: reject unsupported codecs
  vp8: reset loopfilter delta values at keyframes.
  vp56: release frames on error
  vp56: make parse_header return standard error codes
  ivi_common: check that scan pattern is set before using it.
  Update RELEASE file for 0.7.7
  tiffenc: Check av_malloc() results.
  mpegaudiodec: fix short_start calculation
  h264: avoid stuck buffer pointer in decode_nal_units
  yuv4mpeg: return proper error codes.
  smacker audio: sign-extend the initial 16-bit predicted value
  vf_pad: don't give up its own reference to the output buffer.
  avidec: return 0, not packet size from read_packet().
  wmapro: prevent division by zero when sample rate is unspecified
  alsdec: fix number of decoded samples in first sub-block in BGMC mode.
  alsdec: remove dead assignments
  ...

Conflicts:
	RELEASE
	libavformat/avidec.c
	libavformat/yuv4mpeg.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

14 files changed, 80 insertions, 35 deletions

diff --git a/libavcodec/aacdec.c b/libavcodec/aacdec.c
index 5b02d010a3..a0f5344bee 100644
--- a/libavcodec/aacdec.c
+++ b/libavcodec/aacdec.c
@@ -1694,7 +1694,7 @@ static void apply_tns(float coef[1024], TemporalNoiseShaping *tns,
     int w, filt, m, i;
     int bottom, top, order, start, end, size, inc;
     float lpc[TNS_MAX_ORDER];
-    float tmp[TNS_MAX_ORDER];
+    float tmp[TNS_MAX_ORDER + 1];
 
     for (w = 0; w < ics->num_windows; w++) {
         bottom = ics->num_swb;
diff --git a/libavcodec/alsdec.c b/libavcodec/alsdec.c
index 5afad4fa3a..cb679d9955 100644
--- a/libavcodec/alsdec.c
+++ b/libavcodec/alsdec.c
@@ -704,6 +704,10 @@ static int read_var_block_data(ALSDecContext *ctx, ALSBlockData *bd)
                     int rice_param = parcor_rice_table[sconf->coef_table][k][1];
                     int offset     = parcor_rice_table[sconf->coef_table][k][0];
                     quant_cof[k] = decode_rice(gb, rice_param) + offset;
+                    if (quant_cof[k] < -64 || quant_cof[k] > 63) {
+                        av_log(avctx, AV_LOG_ERROR, "quant_cof %d is out of range\n", quant_cof[k]);
+                        return AVERROR_INVALIDDATA;
+                    }
                 }
 
                 // read coefficients 20 to 126
@@ -736,7 +740,7 @@ static int read_var_block_data(ALSDecContext *ctx, ALSBlockData *bd)
             bd->ltp_gain[0]   = decode_rice(gb, 1) << 3;
             bd->ltp_gain[1]   = decode_rice(gb, 2) << 3;
 
-            r                 = get_unary(gb, 0, 4);
+            r                 = get_unary(gb, 0, 3);
             c                 = get_bits(gb, 2);
             bd->ltp_gain[2]   = ltp_gain_values[r][c];
 
@@ -765,7 +769,6 @@ static int read_var_block_data(ALSDecContext *ctx, ALSBlockData *bd)
         int          delta[8];
         unsigned int k    [8];
         unsigned int b = av_clip((av_ceil_log2(bd->block_length) - 3) >> 1, 0, 5);
-        unsigned int i = start;
 
         // read most significant bits
         unsigned int high;
@@ -776,29 +779,30 @@ static int read_var_block_data(ALSDecContext *ctx, ALSBlockData *bd)
 
         current_res = bd->raw_samples + start;
 
-        for (sb = 0; sb < sub_blocks; sb++, i = 0) {
+        for (sb = 0; sb < sub_blocks; sb++) {
+            unsigned int sb_len  = sb_length - (sb ? 0 : start);
+
             k    [sb] = s[sb] > b ? s[sb] - b : 0;
             delta[sb] = 5 - s[sb] + k[sb];
 
-            ff_bgmc_decode(gb, sb_length, current_res,
+            ff_bgmc_decode(gb, sb_len, current_res,
                         delta[sb], sx[sb], &high, &low, &value, ctx->bgmc_lut, ctx->bgmc_lut_status);
 
-            current_res += sb_length;
+            current_res += sb_len;
         }
 
         ff_bgmc_decode_end(gb);
 
 
         // read least significant bits and tails
-        i = start;
         current_res = bd->raw_samples + start;
 
-        for (sb = 0; sb < sub_blocks; sb++, i = 0) {
+        for (sb = 0; sb < sub_blocks; sb++, start = 0) {
             unsigned int cur_tail_code = tail_code[sx[sb]][delta[sb]];
             unsigned int cur_k         = k[sb];
             unsigned int cur_s         = s[sb];
 
-            for (; i < sb_length; i++) {
+            for (; start < sb_length; start++) {
                 int32_t res = *current_res;
 
                 if (res == cur_tail_code) {
diff --git a/libavcodec/flacenc.c b/libavcodec/flacenc.c
index a80f0f9fe3..541ae11341 100644
--- a/libavcodec/flacenc.c
+++ b/libavcodec/flacenc.c
@@ -937,14 +937,16 @@ static int encode_residual_ch(FlacEncodeContext *s, int ch)
         omethod == ORDER_METHOD_8LEVEL) {
         int levels = 1 << omethod;
         uint32_t bits[1 << ORDER_METHOD_8LEVEL];
-        int order;
+        int order       = -1;
         int opt_index   = levels-1;
         opt_order       = max_order-1;
         bits[opt_index] = UINT32_MAX;
         for (i = levels-1; i >= 0; i--) {
+            int last_order = order;
             order = min_order + (((max_order-min_order+1) * (i+1)) / levels)-1;
-            if (order < 0)
-                order = 0;
+            order = av_clip(order, min_order - 1, max_order - 1);
+            if (order == last_order)
+                continue;
             encode_residual_lpc(res, smp, n, order+1, coefs[order], shift[order]);
             bits[i] = find_subframe_rice_params(s, sub, order+1);
             if (bits[i] < bits[opt_index]) {
diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 4c78752a8b..52ffa23391 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -3557,7 +3557,9 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg){
 
                         return 0;
                     }else{
-                        ff_er_add_slice(s, s->resync_mb_x, s->resync_mb_y, s->mb_x, s->mb_y, (AC_END|DC_END|MV_END)&part_mask);
+                        ff_er_add_slice(s, s->resync_mb_x, s->resync_mb_y,
+                                        s->mb_x - 1, s->mb_y,
+                                        (AC_END|DC_END|MV_END)&part_mask);
 
                         return -1;
                     }
@@ -3719,7 +3721,11 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size){
                     break;
             }
 
-            if(buf_index+3 >= buf_size) break;
+
+            if (buf_index + 3 >= buf_size) {
+                buf_index = buf_size;
+                break;
+            }
 
             buf_index+=3;
             if(buf_index >= next_avc) continue;
diff --git a/libavcodec/ivi_common.c b/libavcodec/ivi_common.c
index d47b65d086..d67544c27d 100644
--- a/libavcodec/ivi_common.c
+++ b/libavcodec/ivi_common.c
@@ -410,6 +410,11 @@ int ff_ivi_decode_blocks(GetBitContext *gb, IVIBandDesc *band, IVITile *tile)
             }
 
             if (cbp & 1) { /* block coded ? */
+                if (!band->scan) {
+                    av_log(NULL, AV_LOG_ERROR, "Scan pattern is not set.\n");
+                    return AVERROR_INVALIDDATA;
+                }
+
                 scan_pos = -1;
                 memset(trvec, 0, num_coeffs*sizeof(trvec[0])); /* zero transform vector */
                 memset(col_flags, 0, sizeof(col_flags));      /* zero column flags */
diff --git a/libavcodec/mpegaudiodec.c b/libavcodec/mpegaudiodec.c
index 32dfd23bc1..e5b60e2fb1 100644
--- a/libavcodec/mpegaudiodec.c
+++ b/libavcodec/mpegaudiodec.c
@@ -210,7 +210,7 @@ static void ff_compute_band_indexes(MPADecodeContext *s, GranuleDef *g){
             else
                 g->long_end = 4; /* 8000 Hz */
 
-            g->short_start = 2 + (s->sample_rate_index != 8);
+            g->short_start = 3;
         } else {
             g->long_end = 0;
             g->short_start = 0;
diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c
index ae01b9928b..bf73a79010 100644
--- a/libavcodec/smacker.c
+++ b/libavcodec/smacker.c
@@ -645,7 +645,7 @@ static int smka_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
     }
     if(bits) { //decode 16-bit data
         for(i = stereo; i >= 0; i--)
-            pred[i] = av_bswap16(get_bits(&gb, 16));
+            pred[i] = sign_extend(av_bswap16(get_bits(&gb, 16)), 16);
         for(i = 0; i <= stereo; i++)
             *samples++ = pred[i];
         for(; i < unp_size / 2; i++) {
diff --git a/libavcodec/tiffenc.c b/libavcodec/tiffenc.c
index 97e1dd38c4..a7a1a40dd1 100644
--- a/libavcodec/tiffenc.c
+++ b/libavcodec/tiffenc.c
@@ -305,6 +305,10 @@ static int encode_frame(AVCodecContext * avctx, unsigned char *buf,
 
     strip_sizes = av_mallocz(sizeof(*strip_sizes) * strips);
     strip_offsets = av_mallocz(sizeof(*strip_offsets) * strips);
+    if (!strip_sizes || !strip_offsets) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
 
     bytes_per_row = (((s->width - 1)/s->subsampling[0] + 1) * s->bpp
                     * s->subsampling[0] * s->subsampling[1] + 7) >> 3;
@@ -312,6 +316,7 @@ static int encode_frame(AVCodecContext * avctx, unsigned char *buf,
         yuv_line = av_malloc(bytes_per_row);
         if (yuv_line == NULL){
             av_log(s->avctx, AV_LOG_ERROR, "Not enough memory\n");
+            ret = AVERROR(ENOMEM);
             goto fail;
         }
     }
@@ -324,6 +329,10 @@ static int encode_frame(AVCodecContext * avctx, unsigned char *buf,
 
         zlen = bytes_per_row * s->rps;
         zbuf = av_malloc(zlen);
+        if (!zbuf) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
         strip_offsets[0] = ptr - buf;
         zn = 0;
         for (j = 0; j < s->rps; j++) {
@@ -348,8 +357,13 @@ static int encode_frame(AVCodecContext * avctx, unsigned char *buf,
     } else
 #endif
     {
-        if(s->compr == TIFF_LZW)
+        if (s->compr == TIFF_LZW) {
             s->lzws = av_malloc(ff_lzw_encode_state_size);
+            if (!s->lzws) {
+                ret = AVERROR(ENOMEM);
+                goto fail;
+            }
+        }
         for (i = 0; i < s->height; i++) {
             if (strip_sizes[i / s->rps] == 0) {
                 if(s->compr == TIFF_LZW){
diff --git a/libavcodec/vp5.c b/libavcodec/vp5.c
index 9a89f5d216..ba4a9d290a 100644
--- a/libavcodec/vp5.c
+++ b/libavcodec/vp5.c
@@ -47,18 +47,18 @@ static int vp5_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
     {
         vp56_rac_gets(c, 8);
         if(vp56_rac_gets(c, 5) > 5)
-            return 0;
+            return AVERROR_INVALIDDATA;
         vp56_rac_gets(c, 2);
         if (vp56_rac_get(c)) {
             av_log(s->avctx, AV_LOG_ERROR, "interlacing not supported\n");
-            return 0;
+            return AVERROR_PATCHWELCOME;
         }
         rows = vp56_rac_gets(c, 8);  /* number of stored macroblock rows */
         cols = vp56_rac_gets(c, 8);  /* number of stored macroblock cols */
         if (!rows || !cols) {
             av_log(s->avctx, AV_LOG_ERROR, "Invalid size %dx%d\n",
                    cols << 4, rows << 4);
-            return 0;
+            return AVERROR_INVALIDDATA;
         }
         vp56_rac_gets(c, 8);  /* number of displayed macroblock rows */
         vp56_rac_gets(c, 8);  /* number of displayed macroblock cols */
@@ -67,11 +67,11 @@ static int vp5_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
             16*cols != s->avctx->coded_width ||
             16*rows != s->avctx->coded_height) {
             avcodec_set_dimensions(s->avctx, 16*cols, 16*rows);
-            return 2;
+            return VP56_SIZE_CHANGE;
         }
     } else if (!s->macroblocks)
-        return 0;
-    return 1;
+        return AVERROR_INVALIDDATA;
+    return 0;
 }
 
 static void vp5_parse_vector_adjustment(VP56Context *s, VP56mv *vect)
diff --git a/libavcodec/vp56.c b/libavcodec/vp56.c
index 5b787b6135..7503876d2d 100644
--- a/libavcodec/vp56.c
+++ b/libavcodec/vp56.c
@@ -511,10 +511,16 @@ int ff_vp56_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         s->modelp = &s->models[is_alpha];
 
         res = s->parse_header(s, buf, remaining_buf_size, &golden_frame);
-        if (!res)
-            return -1;
+        if (res < 0) {
+            int i;
+            for (i = 0; i < 4; i++) {
+                if (s->frames[i].data[0])
+                    avctx->release_buffer(avctx, &s->frames[i]);
+            }
+            return res;
+        }
 
-        if (res == 2) {
+        if (res == VP56_SIZE_CHANGE) {
             int i;
             for (i = 0; i < 4; i++) {
                 if (s->frames[i].data[0])
@@ -533,7 +539,7 @@ int ff_vp56_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
                 return -1;
             }
 
-            if (res == 2)
+            if (res == VP56_SIZE_CHANGE)
                 if (vp56_size_changed(avctx)) {
                     avctx->release_buffer(avctx, p);
                     return -1;
diff --git a/libavcodec/vp56.h b/libavcodec/vp56.h
index 4d560dac6a..0bec36fc81 100644
--- a/libavcodec/vp56.h
+++ b/libavcodec/vp56.h
@@ -38,6 +38,8 @@ typedef struct {
     int16_t y;
 } DECLARE_ALIGNED(4, , VP56mv);
 
+#define VP56_SIZE_CHANGE 1
+
 typedef void (*VP56ParseVectorAdjustment)(VP56Context *s,
                                           VP56mv *vect);
 typedef void (*VP56Filter)(VP56Context *s, uint8_t *dst, uint8_t *src,
diff --git a/libavcodec/vp6.c b/libavcodec/vp6.c
index 8294c72dc0..11c763bd83 100644
--- a/libavcodec/vp6.c
+++ b/libavcodec/vp6.c
@@ -50,7 +50,7 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
     int vrt_shift = 0;
     int sub_version;
     int rows, cols;
-    int res = 1;
+    int res = 0;
     int separated_coeff = buf[0] & 1;
 
     s->framep[VP56_FRAME_CURRENT]->key_frame = !(buf[0] & 0x80);
@@ -59,11 +59,11 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
     if (s->framep[VP56_FRAME_CURRENT]->key_frame) {
         sub_version = buf[1] >> 3;
         if (sub_version > 8)
-            return 0;
+            return AVERROR_INVALIDDATA;
         s->filter_header = buf[1] & 0x06;
         if (buf[1] & 1) {
-            av_log(s->avctx, AV_LOG_ERROR, "interlacing not supported\n");
-            return 0;
+            av_log_missing_feature(s->avctx, "Interlacing", 0);
+            return AVERROR_PATCHWELCOME;
         }
         if (separated_coeff || !s->filter_header) {
             coeff_offset = AV_RB16(buf+2) - 2;
@@ -77,7 +77,7 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
         /* buf[5] is number of displayed macroblock cols */
         if (!rows || !cols) {
             av_log(s->avctx, AV_LOG_ERROR, "Invalid size %dx%d\n", cols << 4, rows << 4);
-            return 0;
+            return AVERROR_INVALIDDATA;
         }
 
         if (!s->macroblocks || /* first frame */
@@ -88,7 +88,7 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
                 s->avctx->width  -= s->avctx->extradata[0] >> 4;
                 s->avctx->height -= s->avctx->extradata[0] & 0x0F;
             }
-            res = 2;
+            res = VP56_SIZE_CHANGE;
         }
 
         ff_vp56_init_range_decoder(c, buf+6, buf_size-6);
@@ -100,7 +100,7 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
         s->sub_version = sub_version;
     } else {
         if (!s->sub_version || !s->avctx->coded_width || !s->avctx->coded_height)
-            return 0;
+            return AVERROR_INVALIDDATA;
 
         if (separated_coeff || !s->filter_header) {
             coeff_offset = AV_RB16(buf+1) - 2;
@@ -144,7 +144,7 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size,
         if (buf_size < 0) {
             if (s->framep[VP56_FRAME_CURRENT]->key_frame)
                 avcodec_set_dimensions(s->avctx, 0, 0);
-            return 0;
+            return AVERROR_INVALIDDATA;
         }
         if (s->use_huffman) {
             s->parse_coeff = vp6_parse_coeff_huffman;
diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c
index 3217605e58..24f4d2fce8 100644
--- a/libavcodec/vp8.c
+++ b/libavcodec/vp8.c
@@ -274,6 +274,7 @@ static int decode_frame_header(VP8Context *s, const uint8_t *buf, int buf_size)
         memcpy(s->prob->pred8x8c , vp8_pred8x8c_prob_inter , sizeof(s->prob->pred8x8c));
         memcpy(s->prob->mvc      , vp8_mv_default_prob     , sizeof(s->prob->mvc));
         memset(&s->segmentation, 0, sizeof(s->segmentation));
+        memset(&s->lf_delta, 0, sizeof(s->lf_delta));
     }
 
     if (!s->macroblocks_base || /* first frame */
diff --git a/libavcodec/wmaprodec.c b/libavcodec/wmaprodec.c
index 61d4975a7e..11059a596a 100644
--- a/libavcodec/wmaprodec.c
+++ b/libavcodec/wmaprodec.c
@@ -326,6 +326,11 @@ static av_cold int decode_init(AVCodecContext *avctx)
         return AVERROR_INVALIDDATA;
     }
 
+    if (s->avctx->sample_rate <= 0) {
+        av_log(avctx, AV_LOG_ERROR, "invalid sample rate\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     s->num_channels = avctx->channels;
 
     if (s->num_channels < 0) {