Merge commit 'f82e127dd9c7c0d54bf6400f83c7825e571f9a9e' into release/0.10

* commit 'f82e127dd9c7c0d54bf6400f83c7825e571f9a9e':
  parser: fix large overreads
  dsputil: fix invalid array indexing
  shorten: use the unsigned type where needed
  shorten: report meaningful errors
  shorten: K&R formatting cosmetics
  shorten: set invalid channels count to 0
  matroskadec: request a read buffer for the wav header
  h264: check for luma and chroma bit depth being equal
  vc1: Move init code shared between decoder and parser to common code file.
  libmp3lame: use the correct remaining buffer size when flushing
  xxan: fix invalid memory access in xan_decode_frame_type0()
  wmadec: require block_align to be set.

Conflicts:
	libavcodec/h264.c
	libavcodec/libmp3lame.c
	libavcodec/shorten.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 135 insertions, 121 deletions

diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c
index fc3a35e82a..e9aa93a2d3 100644
--- a/libavcodec/shorten.c
+++ b/libavcodec/shorten.c
@@ -83,7 +83,7 @@ typedef struct ShortenContext {
     GetBitContext gb;
 
     int min_framesize, max_framesize;
-    int channels;
+    unsigned channels;
 
     int32_t *decoded[MAX_CHANNELS];
     int32_t *decoded_base[MAX_CHANNELS];
@@ -108,10 +108,10 @@ typedef struct ShortenContext {
     int got_quit_command;
 } ShortenContext;
 
-static av_cold int shorten_decode_init(AVCodecContext * avctx)
+static av_cold int shorten_decode_init(AVCodecContext *avctx)
 {
     ShortenContext *s = avctx->priv_data;
-    s->avctx = avctx;
+    s->avctx          = avctx;
     avctx->sample_fmt = AV_SAMPLE_FMT_S16;
 
     avcodec_get_frame_defaults(&s->frame);
@@ -126,17 +126,20 @@ static int allocate_buffers(ShortenContext *s)
     int *coeffs;
     void *tmp_ptr;
 
-    for (chan=0; chan<s->channels; chan++) {
-        if(FFMAX(1, s->nmean) >= UINT_MAX/sizeof(int32_t)){
+    for (chan = 0; chan < s->channels; chan++) {
+        if (FFMAX(1, s->nmean) >= UINT_MAX / sizeof(int32_t)) {
             av_log(s->avctx, AV_LOG_ERROR, "nmean too large\n");
-            return -1;
+            return AVERROR_INVALIDDATA;
         }
-        if(s->blocksize + s->nwrap >= UINT_MAX/sizeof(int32_t) || s->blocksize + s->nwrap <= (unsigned)s->nwrap){
-            av_log(s->avctx, AV_LOG_ERROR, "s->blocksize + s->nwrap too large\n");
-            return -1;
+        if (s->blocksize + s->nwrap >= UINT_MAX / sizeof(int32_t) ||
+            s->blocksize + s->nwrap <= (unsigned)s->nwrap) {
+            av_log(s->avctx, AV_LOG_ERROR,
+                   "s->blocksize + s->nwrap too large\n");
+            return AVERROR_INVALIDDATA;
         }
 
-        tmp_ptr = av_realloc(s->offset[chan], sizeof(int32_t)*FFMAX(1, s->nmean));
+        tmp_ptr =
+            av_realloc(s->offset[chan], sizeof(int32_t) * FFMAX(1, s->nmean));
         if (!tmp_ptr)
             return AVERROR(ENOMEM);
         s->offset[chan] = tmp_ptr;
@@ -146,7 +149,7 @@ static int allocate_buffers(ShortenContext *s)
         if (!tmp_ptr)
             return AVERROR(ENOMEM);
         s->decoded_base[chan] = tmp_ptr;
-        for (i=0; i<s->nwrap; i++)
+        for (i = 0; i < s->nwrap; i++)
             s->decoded_base[chan][i] = 0;
         s->decoded[chan] = s->decoded_base[chan] + s->nwrap;
     }
@@ -159,7 +162,6 @@ static int allocate_buffers(ShortenContext *s)
     return 0;
 }
 
-
 static inline unsigned int get_uint(ShortenContext *s, int k)
 {
     if (s->version != 0)
@@ -167,7 +169,6 @@ static inline unsigned int get_uint(ShortenContext *s, int k)
     return get_ur_golomb_shorten(&s->gb, k);
 }
 
-
 static void fix_bitshift(ShortenContext *s, int32_t *buffer)
 {
     int i;
@@ -177,22 +178,20 @@ static void fix_bitshift(ShortenContext *s, int32_t *buffer)
             buffer[i] <<= s->bitshift;
 }
 
-
 static int init_offset(ShortenContext *s)
 {
     int32_t mean = 0;
-    int  chan, i;
+    int chan, i;
     int nblock = FFMAX(1, s->nmean);
     /* initialise offset */
-    switch (s->internal_ftype)
-    {
-        case TYPE_S16HL:
-        case TYPE_S16LH:
-            mean = 0;
-            break;
-        default:
-            av_log(s->avctx, AV_LOG_ERROR, "unknown audio type");
-            return AVERROR_INVALIDDATA;
+    switch (s->internal_ftype) {
+    case TYPE_S16HL:
+    case TYPE_S16LH:
+        mean = 0;
+        break;
+    default:
+        av_log(s->avctx, AV_LOG_ERROR, "unknown audio type");
+        return AVERROR_INVALIDDATA;
     }
 
     for (chan = 0; chan < s->channels; chan++)
@@ -208,21 +207,21 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header,
     short wave_format;
     const uint8_t *end= header + header_size;
 
-    if (bytestream_get_le32(&header) != MKTAG('R','I','F','F')) {
+    if (bytestream_get_le32(&header) != MKTAG('R', 'I', 'F', 'F')) {
         av_log(avctx, AV_LOG_ERROR, "missing RIFF tag\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
-    header += 4; /* chunk size */;
+    header += 4; /* chunk size */
 
-    if (bytestream_get_le32(&header) != MKTAG('W','A','V','E')) {
+    if (bytestream_get_le32(&header) != MKTAG('W', 'A', 'V', 'E')) {
         av_log(avctx, AV_LOG_ERROR, "missing WAVE tag\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
-    while (bytestream_get_le32(&header) != MKTAG('f','m','t',' ')) {
-        len = bytestream_get_le32(&header);
-        if(len<0 || end - header - 8 < len)
+    while (bytestream_get_le32(&header) != MKTAG('f', 'm', 't', ' ')) {
+        len     = bytestream_get_le32(&header);
+        if (len < 0 || end - header - 8 < len)
             return AVERROR_INVALIDDATA;
         header += len;
     }
@@ -230,17 +229,17 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header,
 
     if (len < 16) {
         av_log(avctx, AV_LOG_ERROR, "fmt chunk was too short\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
     wave_format = bytestream_get_le16(&header);
 
     switch (wave_format) {
-        case WAVE_FORMAT_PCM:
-            break;
-        default:
-            av_log(avctx, AV_LOG_ERROR, "unsupported wave format\n");
-            return -1;
+    case WAVE_FORMAT_PCM:
+        break;
+    default:
+        av_log(avctx, AV_LOG_ERROR, "unsupported wave format\n");
+        return AVERROR(ENOSYS);
     }
 
     header += 2;        // skip channels    (already got from shorten header)
@@ -251,7 +250,7 @@ static int decode_wave_header(AVCodecContext *avctx, const uint8_t *header,
 
     if (avctx->bits_per_coded_sample != 16) {
         av_log(avctx, AV_LOG_ERROR, "unsupported number of bits per sample\n");
-        return -1;
+        return AVERROR(ENOSYS);
     }
 
     len -= 16;
@@ -286,11 +285,12 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel,
         /* read/validate prediction order */
         pred_order = get_ur_golomb_shorten(&s->gb, LPCQSIZE);
         if (pred_order > s->nwrap) {
-            av_log(s->avctx, AV_LOG_ERROR, "invalid pred_order %d\n", pred_order);
+            av_log(s->avctx, AV_LOG_ERROR, "invalid pred_order %d\n",
+                   pred_order);
             return AVERROR(EINVAL);
         }
         /* read LPC coefficients */
-        for (i=0; i<pred_order; i++)
+        for (i = 0; i < pred_order; i++)
             s->coeffs[i] = get_sr_golomb_shorten(&s->gb, LPCQUANT);
         coeffs = s->coeffs;
 
@@ -298,7 +298,7 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel,
     } else {
         /* fixed LPC coeffs */
         pred_order = command;
-        coeffs     = fixed_coeffs[pred_order-1];
+        coeffs     = fixed_coeffs[pred_order - 1];
         qshift     = 0;
     }
 
@@ -309,11 +309,12 @@ static int decode_subframe_lpc(ShortenContext *s, int command, int channel,
 
     /* decode residual and do LPC prediction */
     init_sum = pred_order ? (command == FN_QLPC ? s->lpcqoffset : 0) : coffset;
-    for (i=0; i < s->blocksize; i++) {
+    for (i = 0; i < s->blocksize; i++) {
         sum = init_sum;
-        for (j=0; j<pred_order; j++)
-            sum += coeffs[j] * s->decoded[channel][i-j-1];
-        s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) + (sum >> qshift);
+        for (j = 0; j < pred_order; j++)
+            sum += coeffs[j] * s->decoded[channel][i - j - 1];
+        s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) +
+                                 (sum >> qshift);
     }
 
     /* add offset to current samples */
@@ -331,41 +332,47 @@ static int read_header(ShortenContext *s)
     /* shorten signature */
     if (get_bits_long(&s->gb, 32) != AV_RB32("ajkg")) {
         av_log(s->avctx, AV_LOG_ERROR, "missing shorten magic 'ajkg'\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
-    s->lpcqoffset = 0;
-    s->blocksize = DEFAULT_BLOCK_SIZE;
-    s->nmean = -1;
-    s->version = get_bits(&s->gb, 8);
+    s->lpcqoffset     = 0;
+    s->blocksize      = DEFAULT_BLOCK_SIZE;
+    s->nmean          = -1;
+    s->version        = get_bits(&s->gb, 8);
     s->internal_ftype = get_uint(s, TYPESIZE);
 
     s->channels = get_uint(s, CHANSIZE);
+    if (!s->channels) {
+        av_log(s->avctx, AV_LOG_ERROR, "No channels reported\n");
+        return AVERROR_INVALIDDATA;
+    }
     if (s->channels > MAX_CHANNELS) {
         av_log(s->avctx, AV_LOG_ERROR, "too many channels: %d\n", s->channels);
-        return -1;
+        s->channels = 0;
+        return AVERROR_INVALIDDATA;
     }
     s->avctx->channels = s->channels;
 
     /* get blocksize if version > 0 */
     if (s->version > 0) {
-        int skip_bytes, blocksize;
+        int skip_bytes;
+        unsigned blocksize;
 
         blocksize = get_uint(s, av_log2(DEFAULT_BLOCK_SIZE));
         if (!blocksize || blocksize > MAX_BLOCKSIZE) {
-            av_log(s->avctx, AV_LOG_ERROR, "invalid or unsupported block size: %d\n",
+            av_log(s->avctx, AV_LOG_ERROR,
+                   "invalid or unsupported block size: %d\n",
                    blocksize);
             return AVERROR(EINVAL);
         }
         s->blocksize = blocksize;
 
-        maxnlpc = get_uint(s, LPCQSIZE);
+        maxnlpc  = get_uint(s, LPCQSIZE);
         s->nmean = get_uint(s, 0);
 
         skip_bytes = get_uint(s, NSKIPSIZE);
-        for (i=0; i<skip_bytes; i++) {
+        for (i = 0; i < skip_bytes; i++)
             skip_bits(&s->gb, 8);
-        }
     }
     s->nwrap = FFMAX(NWRAP, maxnlpc);
 
@@ -379,21 +386,24 @@ static int read_header(ShortenContext *s)
         s->lpcqoffset = V2LPCQOFFSET;
 
     if (get_ur_golomb_shorten(&s->gb, FNSIZE) != FN_VERBATIM) {
-        av_log(s->avctx, AV_LOG_ERROR, "missing verbatim section at beginning of stream\n");
-        return -1;
+        av_log(s->avctx, AV_LOG_ERROR,
+               "missing verbatim section at beginning of stream\n");
+        return AVERROR_INVALIDDATA;
     }
 
     s->header_size = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE);
-    if (s->header_size >= OUT_BUFFER_SIZE || s->header_size < CANONICAL_HEADER_SIZE) {
-        av_log(s->avctx, AV_LOG_ERROR, "header is wrong size: %d\n", s->header_size);
-        return -1;
+    if (s->header_size >= OUT_BUFFER_SIZE ||
+        s->header_size < CANONICAL_HEADER_SIZE) {
+        av_log(s->avctx, AV_LOG_ERROR, "header is wrong size: %d\n",
+               s->header_size);
+        return AVERROR_INVALIDDATA;
     }
 
-    for (i=0; i<s->header_size; i++)
+    for (i = 0; i < s->header_size; i++)
         s->header[i] = (char)get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE);
 
-    if (decode_wave_header(s->avctx, s->header, s->header_size) < 0)
-        return -1;
+    if ((ret = decode_wave_header(s->avctx, s->header, s->header_size)) < 0)
+        return ret;
 
     s->cur_chan = 0;
     s->bitshift = 0;
@@ -407,15 +417,15 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
                                 int *got_frame_ptr, AVPacket *avpkt)
 {
     const uint8_t *buf = avpkt->data;
-    int buf_size = avpkt->size;
-    ShortenContext *s = avctx->priv_data;
+    int buf_size       = avpkt->size;
+    ShortenContext *s  = avctx->priv_data;
     int i, input_buf_size = 0;
     int ret;
 
     /* allocate internal bitstream buffer */
-    if(s->max_framesize == 0){
+    if (s->max_framesize == 0) {
         void *tmp_ptr;
-        s->max_framesize= 1024; // should hopefully be enough for the first header
+        s->max_framesize = 1024; // should hopefully be enough for the first header
         tmp_ptr = av_fast_realloc(s->bitstream, &s->allocated_bitstream_size,
                                   s->max_framesize);
         if (!tmp_ptr) {
@@ -426,29 +436,32 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
     }
 
     /* append current packet data to bitstream buffer */
-    if(1 && s->max_framesize){//FIXME truncated
-        buf_size= FFMIN(buf_size, s->max_framesize - s->bitstream_size);
-        input_buf_size= buf_size;
-
-        if(s->bitstream_index + s->bitstream_size + buf_size > s->allocated_bitstream_size){
-            memmove(s->bitstream, &s->bitstream[s->bitstream_index], s->bitstream_size);
-            s->bitstream_index=0;
+    if (1 && s->max_framesize) { //FIXME truncated
+        buf_size       = FFMIN(buf_size, s->max_framesize - s->bitstream_size);
+        input_buf_size = buf_size;
+
+        if (s->bitstream_index + s->bitstream_size + buf_size >
+            s->allocated_bitstream_size) {
+            memmove(s->bitstream, &s->bitstream[s->bitstream_index],
+                    s->bitstream_size);
+            s->bitstream_index = 0;
         }
         if (buf)
-            memcpy(&s->bitstream[s->bitstream_index + s->bitstream_size], buf, buf_size);
-        buf= &s->bitstream[s->bitstream_index];
-        buf_size += s->bitstream_size;
-        s->bitstream_size= buf_size;
+            memcpy(&s->bitstream[s->bitstream_index + s->bitstream_size], buf,
+                   buf_size);
+        buf               = &s->bitstream[s->bitstream_index];
+        buf_size         += s->bitstream_size;
+        s->bitstream_size = buf_size;
 
         /* do not decode until buffer has at least max_framesize bytes or
-           the end of the file has been reached */
+         * the end of the file has been reached */
         if (buf_size < s->max_framesize && avpkt->data) {
             *got_frame_ptr = 0;
             return input_buf_size;
         }
     }
     /* init and position bitstream reader */
-    init_get_bits(&s->gb, buf, buf_size*8);
+    init_get_bits(&s->gb, buf, buf_size * 8);
     skip_bits(&s->gb, s->bitindex);
 
     /* process header or next subblock */
@@ -470,7 +483,7 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
         int cmd;
         int len;
 
-        if (get_bits_left(&s->gb) < 3+FNSIZE) {
+        if (get_bits_left(&s->gb) < 3 + FNSIZE) {
             *got_frame_ptr = 0;
             break;
         }
@@ -486,32 +499,32 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
         if (!is_audio_command[cmd]) {
             /* process non-audio command */
             switch (cmd) {
-                case FN_VERBATIM:
-                    len = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE);
-                    while (len--) {
-                        get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE);
-                    }
-                    break;
-                case FN_BITSHIFT:
-                    s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE);
-                    break;
-                case FN_BLOCKSIZE: {
-                    int blocksize = get_uint(s, av_log2(s->blocksize));
-                    if (blocksize > s->blocksize) {
-                        av_log(avctx, AV_LOG_ERROR, "Increasing block size is not supported\n");
-                        return AVERROR_PATCHWELCOME;
-                    }
-                    if (!blocksize || blocksize > MAX_BLOCKSIZE) {
-                        av_log(avctx, AV_LOG_ERROR, "invalid or unsupported "
-                               "block size: %d\n", blocksize);
-                        return AVERROR(EINVAL);
-                    }
-                    s->blocksize = blocksize;
-                    break;
+            case FN_VERBATIM:
+                len = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE);
+                while (len--)
+                    get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE);
+                break;
+            case FN_BITSHIFT:
+                s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE);
+                break;
+            case FN_BLOCKSIZE: {
+                unsigned blocksize = get_uint(s, av_log2(s->blocksize));
+                if (blocksize > s->blocksize) {
+                    av_log(avctx, AV_LOG_ERROR,
+                           "Increasing block size is not supported\n");
+                    return AVERROR_PATCHWELCOME;
                 }
-                case FN_QUIT:
-                    s->got_quit_command = 1;
-                    break;
+                if (!blocksize || blocksize > MAX_BLOCKSIZE) {
+                    av_log(avctx, AV_LOG_ERROR, "invalid or unsupported "
+                                                "block size: %d\n", blocksize);
+                    return AVERROR(EINVAL);
+                }
+                s->blocksize = blocksize;
+                break;
+            }
+            case FN_QUIT:
+                s->got_quit_command = 1;
+                break;
             }
             if (cmd == FN_BLOCKSIZE || cmd == FN_QUIT) {
                 *got_frame_ptr = 0;
@@ -536,7 +549,7 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
                 coffset = s->offset[channel][0];
             else {
                 int32_t sum = (s->version < 2) ? 0 : s->nmean / 2;
-                for (i=0; i<s->nmean; i++)
+                for (i = 0; i < s->nmean; i++)
                     sum += s->offset[channel][i];
                 coffset = sum / s->nmean;
                 if (s->version >= 2)
@@ -545,21 +558,22 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
 
             /* decode samples for this channel */
             if (cmd == FN_ZERO) {
-                for (i=0; i<s->blocksize; i++)
+                for (i = 0; i < s->blocksize; i++)
                     s->decoded[channel][i] = 0;
             } else {
-                if ((ret = decode_subframe_lpc(s, cmd, channel, residual_size, coffset)) < 0)
+                if ((ret = decode_subframe_lpc(s, cmd, channel,
+                                               residual_size, coffset)) < 0)
                     return ret;
             }
 
             /* update means with info from the current block */
             if (s->nmean > 0) {
                 int32_t sum = (s->version < 2) ? 0 : s->blocksize / 2;
-                for (i=0; i<s->blocksize; i++)
+                for (i = 0; i < s->blocksize; i++)
                     sum += s->decoded[channel][i];
 
-                for (i=1; i<s->nmean; i++)
-                    s->offset[channel][i-1] = s->offset[channel][i];
+                for (i = 1; i < s->nmean; i++)
+                    s->offset[channel][i - 1] = s->offset[channel][i];
 
                 if (s->version < 2)
                     s->offset[channel][s->nmean - 1] = sum / s->blocksize;
@@ -568,11 +582,11 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
             }
 
             /* copy wrap samples for use with next block */
-            for (i=-s->nwrap; i<0; i++)
+            for (i = -s->nwrap; i < 0; i++)
                 s->decoded[channel][i] = s->decoded[channel][i + s->blocksize];
 
             /* shift samples to add in unused zero bits which were removed
-               during encoding */
+             * during encoding */
             fix_bitshift(s, s->decoded[channel]);
 
             /* if this is the last channel in the block, output the samples */
@@ -597,13 +611,13 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
         *got_frame_ptr = 0;
 
 finish_frame:
-    s->bitindex = get_bits_count(&s->gb) - 8*((get_bits_count(&s->gb))/8);
-    i= (get_bits_count(&s->gb))/8;
+    s->bitindex = get_bits_count(&s->gb) - 8 * (get_bits_count(&s->gb) / 8);
+    i           = get_bits_count(&s->gb) / 8;
     if (i > buf_size) {
         av_log(s->avctx, AV_LOG_ERROR, "overread: %d\n", i - buf_size);
-        s->bitstream_size=0;
-        s->bitstream_index=0;
-        return -1;
+        s->bitstream_size  = 0;
+        s->bitstream_index = 0;
+        return AVERROR_INVALIDDATA;
     }
     if (s->bitstream_size) {
         s->bitstream_index += i;