Merge remote-tracking branch 'qatar/master'

* qatar/master: (22 commits)
  frwu: Employ more meaningful return values.
  fraps: Use av_fast_padded_malloc() instead of av_realloc()
  mjpegdec: use av_fast_padded_malloc()
  eatqi: use av_fast_padded_malloc()
  asv1: use av_fast_padded_malloc()
  avcodec: Add av_fast_padded_malloc().
  swscale: enable dithering in MMX functions.
  swscale: make rgb24 function macros slightly smaller.
  avcodec.h: Remove some disabled cruft.
  swscale: remove obsolete comment.
  swscale-test: Drop unused argc and argv arguments from main().
  zmbv: Employ more meaningful return values.
  zmbvenc: Employ more meaningful return values.
  vc1: prevent null pointer dereference on broken files
  zmbv: check av_realloc() return values and avoid memleaks on ENOMEM
  truespeech: align buffer
  ac3: Do not read past the end of ff_ac3_band_start_tab.
  dv: Fix small stack overread related to CVE-2011-3929 and CVE-2011-3936.
  dv: Fix null pointer dereference due to ach=0
  dv: check stype
  ...

Conflicts:
	doc/APIchanges
	libavcodec/asv1.c
	libavcodec/avcodec.h
	libavcodec/eatqi.c
	libavcodec/fraps.c
	libavcodec/frwu.c
	libavcodec/zmbv.c
	libavformat/dv.c
	libswscale/swscale.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

11 files changed, 99 insertions, 89 deletions

diff --git a/libavcodec/ac3dsp.c b/libavcodec/ac3dsp.c
index a414db4107..581e5f5071 100644
--- a/libavcodec/ac3dsp.c
+++ b/libavcodec/ac3dsp.c
@@ -109,7 +109,7 @@ static void ac3_bit_alloc_calc_bap_c(int16_t *mask, int16_t *psd,
                                      int snr_offset, int floor,
                                      const uint8_t *bap_tab, uint8_t *bap)
 {
-    int bin, band;
+    int bin, band, band_end;
 
     /* special case, if snr offset is -960, set all bap's to zero */
     if (snr_offset == -960) {
@@ -121,12 +121,14 @@ static void ac3_bit_alloc_calc_bap_c(int16_t *mask, int16_t *psd,
     band = ff_ac3_bin_to_band_tab[start];
     do {
         int m = (FFMAX(mask[band] - snr_offset - floor, 0) & 0x1FE0) + floor;
-        int band_end = FFMIN(ff_ac3_band_start_tab[band+1], end);
+        band_end = ff_ac3_band_start_tab[++band];
+        band_end = FFMIN(band_end, end);
+
         for (; bin < band_end; bin++) {
             int address = av_clip((psd[bin] - m) >> 5, 0, 63);
             bap[bin] = bap_tab[address];
         }
-    } while (end > ff_ac3_band_start_tab[band++]);
+    } while (end > band_end);
 }
 
 static void ac3_update_bap_counts_c(uint16_t mant_cnt[16], uint8_t *bap,
diff --git a/libavcodec/asv1.c b/libavcodec/asv1.c
index 9eeec2675a..0845959d37 100644
--- a/libavcodec/asv1.c
+++ b/libavcodec/asv1.c
@@ -408,7 +408,8 @@ static int decode_frame(AVCodecContext *avctx,
     p->pict_type= AV_PICTURE_TYPE_I;
     p->key_frame= 1;
 
-    av_fast_padded_malloc(&a->bitstream_buffer, &a->bitstream_buffer_size, buf_size);
+    av_fast_padded_malloc(&a->bitstream_buffer, &a->bitstream_buffer_size,
+                          buf_size);
     if (!a->bitstream_buffer)
         return AVERROR(ENOMEM);
 
diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
index 43c2128042..8db04109cd 100644
--- a/libavcodec/avcodec.h
+++ b/libavcodec/avcodec.h
@@ -185,9 +185,6 @@ enum CodecID {
     CODEC_ID_TIERTEXSEQVIDEO,
     CODEC_ID_TIFF,
     CODEC_ID_GIF,
-#if LIBAVCODEC_VERSION_MAJOR == 53
-    CODEC_ID_FFH264,
-#endif
     CODEC_ID_DXA,
     CODEC_ID_DNXHD,
     CODEC_ID_THP,
@@ -205,10 +202,6 @@ enum CodecID {
     CODEC_ID_INDEO5,
     CODEC_ID_MIMIC,
     CODEC_ID_RL2,
-#if LIBAVCODEC_VERSION_MAJOR == 53
-    CODEC_ID_8SVX_EXP,
-    CODEC_ID_8SVX_FIB,
-#endif
     CODEC_ID_ESCAPE124,
     CODEC_ID_DIRAC,
     CODEC_ID_BFI,
@@ -247,18 +240,13 @@ enum CodecID {
     CODEC_ID_DFA,
     CODEC_ID_WMV3IMAGE,
     CODEC_ID_VC1IMAGE,
-#if LIBAVCODEC_VERSION_MAJOR == 53
-    CODEC_ID_G723_1_DEPRECATED,
-    CODEC_ID_G729_DEPRECATED,
-#endif
-    CODEC_ID_UTVIDEO_DEPRECATED,
+    CODEC_ID_UTVIDEO,
     CODEC_ID_BMV_VIDEO,
     CODEC_ID_VBLE,
     CODEC_ID_DXTORY,
     CODEC_ID_V410,
     CODEC_ID_XWD,
     CODEC_ID_Y41P       = MKBETAG('Y','4','1','P'),
-    CODEC_ID_UTVIDEO = 0x800,
     CODEC_ID_ESCAPE130  = MKBETAG('E','1','3','0'),
     CODEC_ID_AVRP       = MKBETAG('A','V','R','P'),
 
@@ -397,15 +385,11 @@ enum CodecID {
     CODEC_ID_AAC_LATM,
     CODEC_ID_QDMC,
     CODEC_ID_CELT,
-#if LIBAVCODEC_VERSION_MAJOR > 53
-    CODEC_ID_G723_1_DEPRECATED,
-    CODEC_ID_G729_DEPRECATED,
+    CODEC_ID_G723_1,
+    CODEC_ID_G729,
     CODEC_ID_8SVX_EXP,
     CODEC_ID_8SVX_FIB,
-#endif
     CODEC_ID_BMV_AUDIO,
-    CODEC_ID_G729 = 0x15800,
-    CODEC_ID_G723_1= 0x15801,
     CODEC_ID_FFWAVESYNTH = MKBETAG('F','F','W','S'),
     CODEC_ID_8SVX_RAW   = MKBETAG('8','S','V','X'),
     CODEC_ID_SONIC       = MKBETAG('S','O','N','C'),
@@ -1523,7 +1507,6 @@ typedef struct AVCodecContext {
 #define FF_BUG_DC_CLIP          4096
 #define FF_BUG_MS               8192 ///< Work around various bugs in Microsoft's broken decoders.
 #define FF_BUG_TRUNCATED       16384
-//#define FF_BUG_FAKE_SCALABILITY 16 //Autodetection should work 100%.
 
     /**
      * luma single coefficient elimination threshold
@@ -2096,14 +2079,6 @@ typedef struct AVCodecContext {
      * - decoding: unused
      */
     int context_model;
-#if 0
-    /**
-     *
-     * - encoding: unused
-     * - decoding: Set by user.
-     */
-    uint8_t * (*realloc)(struct AVCodecContext *s, uint8_t *buf, int buf_size);
-#endif
 
     /**
      * slice flags
diff --git a/libavcodec/eatqi.c b/libavcodec/eatqi.c
index d650a71713..2d6cff58d4 100644
--- a/libavcodec/eatqi.c
+++ b/libavcodec/eatqi.c
@@ -127,7 +127,8 @@ static int tqi_decode_frame(AVCodecContext *avctx,
         return -1;
     }
 
-    av_fast_padded_malloc(&t->bitstream_buf, &t->bitstream_buf_size, buf_end-buf);
+    av_fast_padded_malloc(&t->bitstream_buf, &t->bitstream_buf_size,
+                          buf_end - buf);
     if (!t->bitstream_buf)
         return AVERROR(ENOMEM);
     s->dsp.bswap_buf(t->bitstream_buf, (const uint32_t*)buf, (buf_end-buf)/4);
diff --git a/libavcodec/frwu.c b/libavcodec/frwu.c
index b4394a3044..b47cb717f2 100644
--- a/libavcodec/frwu.c
+++ b/libavcodec/frwu.c
@@ -28,7 +28,7 @@ static av_cold int decode_init(AVCodecContext *avctx)
 {
     if (avctx->width & 1) {
         av_log(avctx, AV_LOG_ERROR, "frwu needs even width\n");
-        return -1;
+        return AVERROR(EINVAL);
     }
     avctx->pix_fmt = PIX_FMT_UYVY422;
 
@@ -42,7 +42,7 @@ static av_cold int decode_init(AVCodecContext *avctx)
 static int decode_frame(AVCodecContext *avctx, void *data, int *data_size,
                         AVPacket *avpkt)
 {
-    int field;
+    int field, ret;
     AVFrame *pic = avctx->coded_frame;
     const uint8_t *buf = avpkt->data;
     const uint8_t *buf_end = buf + avpkt->size;
@@ -52,16 +52,18 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size,
 
     if (avpkt->size < avctx->width * 2 * avctx->height + 4 + 2*8) {
         av_log(avctx, AV_LOG_ERROR, "Packet is too small.\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
     if (bytestream_get_le32(&buf) != AV_RL32("FRW1")) {
         av_log(avctx, AV_LOG_ERROR, "incorrect marker\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
     pic->reference = 0;
-    if (avctx->get_buffer(avctx, pic) < 0)
-        return -1;
+    if ((ret = avctx->get_buffer(avctx, pic)) < 0) {
+        av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
+        return ret;
+    }
 
     pic->pict_type = AV_PICTURE_TYPE_I;
     pic->key_frame = 1;
@@ -74,16 +76,16 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         int field_size, min_field_size = avctx->width * 2 * field_h;
         uint8_t *dst = pic->data[0];
         if (buf_end - buf < 8)
-            return -1;
+            return AVERROR_INVALIDDATA;
         buf += 4; // flags? 0x80 == bottom field maybe?
         field_size = bytestream_get_le32(&buf);
         if (field_size < min_field_size) {
             av_log(avctx, AV_LOG_ERROR, "Field size %i is too small (required %i)\n", field_size, min_field_size);
-            return -1;
+            return AVERROR_INVALIDDATA;
         }
         if (buf_end - buf < field_size) {
             av_log(avctx, AV_LOG_ERROR, "Packet is too small, need %i, have %i\n", field_size, (int)(buf_end - buf));
-            return -1;
+            return AVERROR_INVALIDDATA;
         }
         if (field)
             dst += pic->linesize[0];
diff --git a/libavcodec/golomb-test.c b/libavcodec/golomb-test.c
index c13866bbd7..54644ade46 100644
--- a/libavcodec/golomb-test.c
+++ b/libavcodec/golomb-test.c
@@ -21,52 +21,80 @@
 #include <stdint.h>
 #include <stdio.h>
 
-#include "avcodec.h"
-#include "dsputil.h"
+#include "libavutil/mem.h"
+
 #include "get_bits.h"
 #include "golomb.h"
 #include "put_bits.h"
 
-#undef printf
-#define COUNT 8000
-#define SIZE (COUNT * 40)
+#undef fprintf
+#define COUNT 8191
+#define SIZE (COUNT * 4)
 
 int main(void)
 {
-    int i;
-    uint8_t temp[SIZE];
+    int i, ret = 0;
+    uint8_t *temp;
     PutBitContext pb;
     GetBitContext gb;
 
+    temp = av_malloc(SIZE);
+    if (!temp)
+        return 2;
+
     init_put_bits(&pb, temp, SIZE);
-    printf("testing unsigned exp golomb\n");
     for (i = 0; i < COUNT; i++)
         set_ue_golomb(&pb, i);
     flush_put_bits(&pb);
 
     init_get_bits(&gb, temp, 8 * SIZE);
     for (i = 0; i < COUNT; i++) {
-        int j, s = show_bits(&gb, 24);
+        int j, s = show_bits(&gb, 25);
 
         j = get_ue_golomb(&gb);
-        if (j != i)
-            printf("mismatch at %d (%d should be %d) bits: %6X\n", i, j, i, s);
+        if (j != i) {
+            fprintf(stderr, "get_ue_golomb: expected %d, got %d. bits: %7x\n",
+                    i, j, s);
+            ret = 1;
+        }
+    }
+
+#define EXTEND(i) (i << 3 | i & 7)
+    init_put_bits(&pb, temp, SIZE);
+    for (i = 0; i < COUNT; i++)
+        set_ue_golomb(&pb, EXTEND(i));
+    flush_put_bits(&pb);
+
+    init_get_bits(&gb, temp, 8 * SIZE);
+    for (i = 0; i < COUNT; i++) {
+        int j, s = show_bits_long(&gb, 32);
+
+        j = get_ue_golomb_long(&gb);
+        if (j != EXTEND(i)) {
+            fprintf(stderr, "get_ue_golomb_long: expected %d, got %d. "
+                    "bits: %8x\n", EXTEND(i), j, s);
+            ret = 1;
+        }
     }
 
     init_put_bits(&pb, temp, SIZE);
-    printf("testing signed exp golomb\n");
     for (i = 0; i < COUNT; i++)
         set_se_golomb(&pb, i - COUNT / 2);
     flush_put_bits(&pb);
 
     init_get_bits(&gb, temp, 8 * SIZE);
     for (i = 0; i < COUNT; i++) {
-        int j, s = show_bits(&gb, 24);
+        int j, s = show_bits(&gb, 25);
 
         j = get_se_golomb(&gb);
-        if (j != i - COUNT / 2)
-            printf("mismatch at %d (%d should be %d) bits: %6X\n", i, j, i, s);
+        if (j != i - COUNT / 2) {
+            fprintf(stderr, "get_se_golomb: expected %d, got %d. bits: %7x\n",
+                    i - COUNT / 2, j, s);
+            ret = 1;
+        }
     }
 
-    return 0;
+    av_free(temp);
+
+    return ret;
 }
diff --git a/libavcodec/truespeech.c b/libavcodec/truespeech.c
index 9bbbf5172e..c2c68abd6f 100644
--- a/libavcodec/truespeech.c
+++ b/libavcodec/truespeech.c
@@ -37,7 +37,7 @@ typedef struct {
     AVFrame frame;
     DSPContext dsp;
     /* input data */
-    uint8_t buffer[32];
+    DECLARE_ALIGNED(16, uint8_t, buffer)[32];
     int16_t vector[8];  ///< input vector: 5/5/4/4/4/3/3/3
     int offset1[2];     ///< 8-bit value, used in one copying offset
     int offset2[4];     ///< 7-bit value, encodes offsets for copying and for two-point filter
diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index 0df3d7f8b0..f21e36fd9c 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -91,7 +91,7 @@ void av_fast_padded_malloc(void *ptr, unsigned int *size, size_t min_size)
 {
     uint8_t **p = ptr;
     if (min_size > SIZE_MAX - FF_INPUT_BUFFER_PADDING_SIZE) {
-        *p = NULL;
+        av_freep(p);
         *size = 0;
         return;
     }
@@ -1448,9 +1448,9 @@ av_cold int avcodec_close(AVCodecContext *avctx)
 static enum CodecID remap_deprecated_codec_id(enum CodecID id)
 {
     switch(id){
-        case CODEC_ID_G723_1_DEPRECATED : return CODEC_ID_G723_1;
-        case CODEC_ID_G729_DEPRECATED   : return CODEC_ID_G729;
-        case CODEC_ID_UTVIDEO_DEPRECATED: return CODEC_ID_UTVIDEO;
+        //This is for future deprecatec codec ids, its empty since
+        //last major bump but will fill up again over time, please dont remove it
+//         case CODEC_ID_UTVIDEO_DEPRECATED: return CODEC_ID_UTVIDEO;
         default                         : return id;
     }
 }
diff --git a/libavcodec/vc1dec.c b/libavcodec/vc1dec.c
index 177c5082bb..a3ac2cbbaa 100644
--- a/libavcodec/vc1dec.c
+++ b/libavcodec/vc1dec.c
@@ -5712,7 +5712,7 @@ static int vc1_decode_frame(AVCodecContext *avctx, void *data,
             if (!v->field_mode || v->second_field)
                 s->end_mb_y = (i == n_slices     ) ? mb_height : FFMIN(mb_height, slices[i].mby_start % mb_height);
             else
-                s->end_mb_y = (i == n_slices1 + 1) ? mb_height : FFMIN(mb_height, slices[i].mby_start % mb_height);
+                s->end_mb_y = (i <= n_slices1 + 1) ? mb_height : FFMIN(mb_height, slices[i].mby_start % mb_height);
             vc1_decode_blocks(v);
             if (i != n_slices)
                 s->gb = slices[i].gb;
diff --git a/libavcodec/zmbv.c b/libavcodec/zmbv.c
index 22a3272dee..3797cfb4ae 100644
--- a/libavcodec/zmbv.c
+++ b/libavcodec/zmbv.c
@@ -403,16 +403,17 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
     ZmbvContext * const c = avctx->priv_data;
     int zret = Z_OK; // Zlib return code
     int len = buf_size;
-    int hi_ver, lo_ver;
+    int hi_ver, lo_ver, ret;
+    uint8_t *tmp;
 
     if (c->pic.data[0])
             avctx->release_buffer(avctx, &c->pic);
 
     c->pic.reference = 3;
     c->pic.buffer_hints = FF_BUFFER_HINTS_VALID;
-    if (avctx->get_buffer(avctx, &c->pic) < 0) {
+    if ((ret = avctx->get_buffer(avctx, &c->pic)) < 0) {
         av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
-        return -1;
+        return ret;
     }
 
     /* parse header */
@@ -434,19 +435,19 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
                "Flags=%X ver=%i.%i comp=%i fmt=%i blk=%ix%i\n",
                c->flags,hi_ver,lo_ver,c->comp,c->fmt,c->bw,c->bh);
         if (hi_ver != 0 || lo_ver != 1) {
-            av_log(avctx, AV_LOG_ERROR, "Unsupported version %i.%i\n",
-            hi_ver, lo_ver);
-            return -1;
+            av_log_ask_for_sample(avctx, "Unsupported version %i.%i\n",
+                                  hi_ver, lo_ver);
+            return AVERROR_PATCHWELCOME;
         }
         if (c->bw == 0 || c->bh == 0) {
-            av_log(avctx, AV_LOG_ERROR, "Unsupported block size %ix%i\n",
-                   c->bw, c->bh);
-            return -1;
+            av_log_ask_for_sample(avctx, "Unsupported block size %ix%i\n",
+                                  c->bw, c->bh);
+            return AVERROR_PATCHWELCOME;
         }
         if (c->comp != 0 && c->comp != 1) {
-            av_log(avctx, AV_LOG_ERROR, "Unsupported compression type %i\n",
-                   c->comp);
-            return -1;
+            av_log_ask_for_sample(avctx, "Unsupported compression type %i\n",
+                                  c->comp);
+            return AVERROR_PATCHWELCOME;
         }
 
         switch (c->fmt) {
@@ -475,9 +476,9 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
             break;
         default:
             c->decode_xor = NULL;
-            av_log(avctx, AV_LOG_ERROR,
-                   "Unsupported (for now) format %i\n", c->fmt);
-            return -1;
+            av_log_ask_for_sample(avctx, "Unsupported (for now) format %i\n",
+                                  c->fmt);
+            return AVERROR_PATCHWELCOME;
         }
 
         zret = inflateReset(&c->zstream);
@@ -495,10 +496,10 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
         c->decode_intra= decode_intra;
     }
 
-    if (c->decode_intra == NULL) {
-        av_log(avctx, AV_LOG_ERROR, "Error! Got no format or no keyframe!\n");
-        return -1;
-    }
+     if (c->decode_intra == NULL) {
+         av_log(avctx, AV_LOG_ERROR, "Error! Got no format or no keyframe!\n");
+         return AVERROR_INVALIDDATA;
+     }
 
     if (c->comp == 0) { //Uncompressed data
         memcpy(c->decomp_buf, buf, len);
@@ -628,7 +629,7 @@ static av_cold int decode_init(AVCodecContext *avctx)
         if ((c->decomp_buf = av_malloc(c->decomp_size)) == NULL) {
             av_log(avctx, AV_LOG_ERROR,
                    "Can't allocate decompression buffer.\n");
-            return 1;
+            return AVERROR(ENOMEM);
         }
     }
 
@@ -638,7 +639,7 @@ static av_cold int decode_init(AVCodecContext *avctx)
     zret = inflateInit(&c->zstream);
     if (zret != Z_OK) {
         av_log(avctx, AV_LOG_ERROR, "Inflate init error: %d\n", zret);
-        return 1;
+        return -1;
     }
 
     return 0;
diff --git a/libavcodec/zmbvenc.c b/libavcodec/zmbvenc.c
index 4b51fb167d..3211e6ff4d 100644
--- a/libavcodec/zmbvenc.c
+++ b/libavcodec/zmbvenc.c
@@ -265,7 +265,7 @@ static av_cold int encode_init(AVCodecContext *avctx)
         lvl = avctx->compression_level;
     if(lvl < 0 || lvl > 9){
         av_log(avctx, AV_LOG_ERROR, "Compression level should be 0-9, not %i\n", lvl);
-        return -1;
+        return AVERROR(EINVAL);
     }
 
     // Needed if zlib unused or init aborted before deflateInit
@@ -274,7 +274,7 @@ static av_cold int encode_init(AVCodecContext *avctx)
         ((avctx->width + ZMBV_BLOCK - 1) / ZMBV_BLOCK) * ((avctx->height + ZMBV_BLOCK - 1) / ZMBV_BLOCK) * 2 + 4;
     if ((c->work_buf = av_malloc(c->comp_size)) == NULL) {
         av_log(avctx, AV_LOG_ERROR, "Can't allocate work buffer.\n");
-        return -1;
+        return AVERROR(ENOMEM);
     }
     /* Conservative upper bound taken from zlib v1.2.1 source via lcl.c */
     c->comp_size = c->comp_size + ((c->comp_size + 7) >> 3) +
@@ -283,12 +283,12 @@ static av_cold int encode_init(AVCodecContext *avctx)
     /* Allocate compression buffer */
     if ((c->comp_buf = av_malloc(c->comp_size)) == NULL) {
         av_log(avctx, AV_LOG_ERROR, "Can't allocate compression buffer.\n");
-        return -1;
+        return AVERROR(ENOMEM);
     }
     c->pstride = FFALIGN(avctx->width, 16);
     if ((c->prev = av_malloc(c->pstride * avctx->height)) == NULL) {
         av_log(avctx, AV_LOG_ERROR, "Can't allocate picture.\n");
-        return -1;
+        return AVERROR(ENOMEM);
     }
 
     c->zstream.zalloc = Z_NULL;