Merge remote-tracking branch 'qatar/master'

* qatar/master:
  sws: implement MMX/SSE2/SSSE3/SSE4 versions for horizontal scaling.
  include stdint.h in adpcm_data.h
  mpeg12: reorder functions to avoid ugly forward declarations
  Fixed off by one packet size allocation in the smacker demuxer.
  Check for invalid packet size in the smacker demuxer.
  ape demuxer: fix segfault on memory allocation failure.
  xan: Add some buffer checks
  xan: Remove extra trailing newline
  Fixed size given to init_get_bits() in xan decoder.

Conflicts:
	libavcodec/mpeg12.c
	libswscale/x86/swscale_template.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

1 files changed, 33 insertions, 11 deletions

diff --git a/libavcodec/xan.c b/libavcodec/xan.c
index 598d1e1423..3e950f1466 100644
--- a/libavcodec/xan.c
+++ b/libavcodec/xan.c
@@ -97,17 +97,21 @@ static av_cold int xan_decode_init(AVCodecContext *avctx)
     return 0;
 }
 
-static int xan_huffman_decode(unsigned char *dest, const unsigned char *src,
-    int dest_len)
+static int xan_huffman_decode(unsigned char *dest, int dest_len,
+                              const unsigned char *src, int src_len)
 {
     unsigned char byte = *src++;
     unsigned char ival = byte + 0x16;
     const unsigned char * ptr = src + byte*2;
+    int ptr_len = src_len - 1 - byte*2;
     unsigned char val = ival;
     unsigned char *dest_end = dest + dest_len;
     GetBitContext gb;
 
-    init_get_bits(&gb, ptr, 0); // FIXME: no src size available
+    if (ptr_len < 0)
+        return AVERROR_INVALIDDATA;
+
+    init_get_bits(&gb, ptr, ptr_len * 8);
 
     while ( val != 0x16 ) {
         val = src[val - 0x17 + get_bits1(&gb) * byte];
@@ -246,7 +250,7 @@ static inline void xan_wc3_copy_pixel_run(XanContext *s,
     }
 }
 
-static void xan_wc3_decode_frame(XanContext *s) {
+static int xan_wc3_decode_frame(XanContext *s) {
 
     int width = s->avctx->width;
     int height = s->avctx->height;
@@ -266,13 +270,30 @@ static void xan_wc3_decode_frame(XanContext *s) {
     const unsigned char *size_segment;
     const unsigned char *vector_segment;
     const unsigned char *imagedata_segment;
+    int huffman_offset, size_offset, vector_offset, imagedata_offset;
+
+    if (s->size < 8)
+        return AVERROR_INVALIDDATA;
+
+    huffman_offset    = AV_RL16(&s->buf[0]);
+    size_offset       = AV_RL16(&s->buf[2]);
+    vector_offset     = AV_RL16(&s->buf[4]);
+    imagedata_offset  = AV_RL16(&s->buf[6]);
 
-    huffman_segment =   s->buf + AV_RL16(&s->buf[0]);
-    size_segment =      s->buf + AV_RL16(&s->buf[2]);
-    vector_segment =    s->buf + AV_RL16(&s->buf[4]);
-    imagedata_segment = s->buf + AV_RL16(&s->buf[6]);
+    if (huffman_offset   >= s->size ||
+        size_offset      >= s->size ||
+        vector_offset    >= s->size ||
+        imagedata_offset >= s->size)
+        return AVERROR_INVALIDDATA;
 
-    xan_huffman_decode(opcode_buffer, huffman_segment, opcode_buffer_size);
+    huffman_segment   = s->buf + huffman_offset;
+    size_segment      = s->buf + size_offset;
+    vector_segment    = s->buf + vector_offset;
+    imagedata_segment = s->buf + imagedata_offset;
+
+    if (xan_huffman_decode(opcode_buffer, opcode_buffer_size,
+                           huffman_segment, s->size - huffman_offset) < 0)
+        return AVERROR_INVALIDDATA;
 
     if (imagedata_segment[0] == 2)
         xan_unpack(s->buffer2, &imagedata_segment[1], s->buffer2_size);
@@ -358,6 +379,7 @@ static void xan_wc3_decode_frame(XanContext *s) {
         y += (x + size) / width;
         x  = (x + size) % width;
     }
+    return 0;
 }
 
 #if RUNTIME_GAMMA
@@ -519,7 +541,8 @@ static int xan_decode_frame(AVCodecContext *avctx,
     s->buf = buf;
     s->size = buf_size;
 
-    xan_wc3_decode_frame(s);
+    if (xan_wc3_decode_frame(s) < 0)
+        return AVERROR_INVALIDDATA;
 
     /* release the last frame if it is allocated */
     if (s->last_frame.data[0])
@@ -563,4 +586,3 @@ AVCodec ff_xan_wc3_decoder = {
     .capabilities   = CODEC_CAP_DR1,
     .long_name = NULL_IF_CONFIG_SMALL("Wing Commander III / Xan"),
 };
-