Merge remote-tracking branch 'qatar/master'

* qatar/master: (34 commits)
  h264: reset h->ref_count in case of errors in ff_h264_decode_ref_pic_list_reordering()
  error_resilience: fix the check for missing references in ff_er_frame_end() for H264
  4xm: prevent NULL dereference with invalid huffman table
  4xmdemux: prevent use of uninitialized memory
  4xm: clear FF_INPUT_BUFFER_PADDING_SIZE bytes in temporary buffers
  ptx: check for out of bound reads
  tiffdec: fix out of bound reads/writes
  eacmv: check for out of bound reads
  eacmv: fix potential pointer arithmetic overflows
  adpcm: fix out of bound reads due to integer overflow
  anm: prevent infinite loop
  avsdemux: check for out of bound writes
  avs: check for out of bound reads
  avsdemux: check for corrupted data
  AVOptions: refactor set_number/write_number
  AVOptions: cosmetics, rename static av_set_number2() to write_number().
  AVOptions: cosmetics, move and rename static av_set_number().
  AVOptions: split av_set_string3 into opt type-specific functions
  avidec: fix signed overflow in avi_sync()
  mxfdec: Fix some buffer overreads caused by the misuse of AVPacket related functions.
  ...

Conflicts:
	Changelog
	configure
	libavcodec/ptx.c
	libavcodec/ra144.c
	libavcodec/vaapi_vc1.c
	libavcodec/vc1.c
	libavcodec/version.h
	libavformat/4xm.c
	libavformat/avidec.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

3 files changed, 21 insertions, 12 deletions

diff --git a/libavformat/4xm.c b/libavformat/4xm.c
index f535c49420..b95eac71bd 100644
--- a/libavformat/4xm.c
+++ b/libavformat/4xm.c
@@ -177,7 +177,7 @@ static int fourxm_read_header(AVFormatContext *s,
                                               sizeof(AudioTrack),
                                               current_track + 1);
                 if (!fourxm->tracks) {
-                    ret=  AVERROR(ENOMEM);
+                    ret = AVERROR(ENOMEM);
                     goto fail;
                 }
                 memset(&fourxm->tracks[fourxm->track_count], 0,
diff --git a/libavformat/avidec.c b/libavformat/avidec.c
index 8dce8f8c45..acb73aff19 100644
--- a/libavformat/avidec.c
+++ b/libavformat/avidec.c
@@ -874,12 +874,13 @@ static int avi_sync(AVFormatContext *s, int exit_early)
 {
     AVIContext *avi = s->priv_data;
     AVIOContext *pb = s->pb;
-    int n, d[8];
+    int n;
+    unsigned int d[8];
     unsigned int size;
     int64_t i, sync;
 
 start_sync:
-    memset(d, -1, sizeof(int)*8);
+    memset(d, -1, sizeof(d));
     for(i=sync=avio_tell(pb); !url_feof(pb); i++) {
         int j;
 
@@ -891,7 +892,7 @@ start_sync:
 
         n= get_stream_idx(d+2);
 //av_log(s, AV_LOG_DEBUG, "%X %X %X %X %X %X %X %X %"PRId64" %d %d\n", d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], i, size, n);
-        if(i + (uint64_t)size > avi->fsize || d[0]<0)
+        if(i + (uint64_t)size > avi->fsize || d[0] > 127)
             continue;
 
         //parse ix##
diff --git a/libavformat/mxfdec.c b/libavformat/mxfdec.c
index 8ec4a7032d..5967df08c6 100644
--- a/libavformat/mxfdec.c
+++ b/libavformat/mxfdec.c
@@ -256,12 +256,13 @@ static int mxf_get_d10_aes3_packet(AVIOContext *pb, AVStream *st, AVPacket *pkt,
 
     if (length > 61444) /* worst case PAL 1920 samples 8 channels */
         return -1;
-    av_new_packet(pkt, length);
-    avio_read(pb, pkt->data, length);
+    length = av_get_packet(pb, pkt, length);
+    if (length < 0)
+        return length;
     data_ptr = pkt->data;
     end_ptr = pkt->data + length;
     buf_ptr = pkt->data + 4; /* skip SMPTE 331M header */
-    for (; buf_ptr < end_ptr; ) {
+    for (; buf_ptr + st->codec->channels*4 < end_ptr; ) {
         for (i = 0; i < st->codec->channels; i++) {
             uint32_t sample = bytestream_get_le32(&buf_ptr);
             if (st->codec->bits_per_coded_sample == 24)
@@ -271,7 +272,7 @@ static int mxf_get_d10_aes3_packet(AVIOContext *pb, AVStream *st, AVPacket *pkt,
         }
         buf_ptr += 32 - st->codec->channels*4; // always 8 channels stored SMPTE 331M
     }
-    pkt->size = data_ptr - pkt->data;
+    av_shrink_packet(pkt, data_ptr - pkt->data);
     return 0;
 }
 
@@ -323,12 +324,16 @@ static int mxf_decrypt_triplet(AVFormatContext *s, AVPacket *pkt, KLVPacket *klv
     if (memcmp(tmpbuf, checkv, 16))
         av_log(s, AV_LOG_ERROR, "probably incorrect decryption key\n");
     size -= 32;
-    av_get_packet(pb, pkt, size);
+    size = av_get_packet(pb, pkt, size);
+    if (size < 0)
+        return size;
+    else if (size < plaintext_size)
+        return AVERROR_INVALIDDATA;
     size -= plaintext_size;
     if (mxf->aesc)
         av_aes_crypt(mxf->aesc, &pkt->data[plaintext_size],
                      &pkt->data[plaintext_size], size >> 4, ivec, 1);
-    pkt->size = orig_size;
+    av_shrink_packet(pkt, orig_size);
     pkt->stream_index = index;
     avio_skip(pb, end - avio_tell(pb));
     return 0;
@@ -365,8 +370,11 @@ static int mxf_read_packet(AVFormatContext *s, AVPacket *pkt)
                     av_log(s, AV_LOG_ERROR, "error reading D-10 aes3 frame\n");
                     return -1;
                 }
-            } else
-                av_get_packet(s->pb, pkt, klv.length);
+            } else {
+                int ret = av_get_packet(s->pb, pkt, klv.length);
+                if (ret < 0)
+                    return ret;
+            }
             pkt->stream_index = index;
             pkt->pos = klv.offset;
             return 0;