Merge commit 'a1c525f7eb0783d31ba7a653865b6cbd3dc880de'

* commit 'a1c525f7eb0783d31ba7a653865b6cbd3dc880de':
  pcx: return meaningful error codes.
  tmv: return meaningful error codes.
  msrle: return meaningful error codes.
  cscd: return meaningful error codes.
  yadif: x86: fix build for compilers without aligned stack
  lavc: introduce the convenience function init_get_bits8
  lavc: check for overflow in init_get_bits

Conflicts:
	libavcodec/cscd.c
	libavcodec/pcx.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

6 files changed, 81 insertions, 42 deletions

diff --git a/libavcodec/cscd.c b/libavcodec/cscd.c
index e7ebb37f58..110b06fa2b 100644
--- a/libavcodec/cscd.c
+++ b/libavcodec/cscd.c
@@ -68,18 +68,19 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     int buf_size = avpkt->size;
     CamStudioContext *c = avctx->priv_data;
     AVFrame *picture = data;
+    int ret;
 
     if (buf_size < 2) {
         av_log(avctx, AV_LOG_ERROR, "coded frame too small\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
     c->pic.reference = 3;
     c->pic.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_READABLE |
                           FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE;
-    if (avctx->reget_buffer(avctx, &c->pic) < 0) {
+    if ((ret = avctx->reget_buffer(avctx, &c->pic)) < 0) {
         av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
-        return -1;
+        return ret;
     }
 
     // decompress data
@@ -98,12 +99,12 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
             break;
 #else
             av_log(avctx, AV_LOG_ERROR, "compiled without zlib support\n");
-            return -1;
+            return AVERROR(ENOSYS);
 #endif
         }
         default:
             av_log(avctx, AV_LOG_ERROR, "unknown compression\n");
-            return -1;
+            return AVERROR_INVALIDDATA;
     }
 
     // flip upside down, add difference frame
diff --git a/libavcodec/get_bits.h b/libavcodec/get_bits.h
index 777176dd30..af7e156e59 100644
--- a/libavcodec/get_bits.h
+++ b/libavcodec/get_bits.h
@@ -366,25 +366,49 @@ static inline int check_marker(GetBitContext *s, const char *msg)
 }
 
 /**
- * Inititalize GetBitContext.
- * @param buffer bitstream buffer, must be FF_INPUT_BUFFER_PADDING_SIZE bytes larger than the actual read bits
- * because some optimized bitstream readers read 32 or 64 bit at once and could read over the end
+ * Initialize GetBitContext.
+ * @param buffer bitstream buffer, must be FF_INPUT_BUFFER_PADDING_SIZE bytes
+ *        larger than the actual read bits because some optimized bitstream
+ *        readers read 32 or 64 bit at once and could read over the end
  * @param bit_size the size of the buffer in bits
+ * @return 0 on success, AVERROR_INVALIDDATA if the buffer_size would overflow.
  */
-static inline void init_get_bits(GetBitContext *s, const uint8_t *buffer,
-                                 int bit_size)
+static inline int init_get_bits(GetBitContext *s, const uint8_t *buffer,
+                                int bit_size)
 {
-    int buffer_size = (bit_size+7)>>3;
-    if (buffer_size < 0 || bit_size < 0) {
+    int buffer_size;
+    int ret = 0;
+
+    if (bit_size > INT_MAX - 7 || bit_size < 0) {
         buffer_size = bit_size = 0;
         buffer = NULL;
+        ret = AVERROR_INVALIDDATA;
     }
 
+    buffer_size = (bit_size + 7) >> 3;
+
     s->buffer       = buffer;
     s->size_in_bits = bit_size;
     s->size_in_bits_plus8 = bit_size + 8;
     s->buffer_end   = buffer + buffer_size;
     s->index        = 0;
+    return ret;
+}
+
+/**
+ * Initialize GetBitContext.
+ * @param buffer bitstream buffer, must be FF_INPUT_BUFFER_PADDING_SIZE bytes
+ *        larger than the actual read bits because some optimized bitstream
+ *        readers read 32 or 64 bit at once and could read over the end
+ * @param byte_size the size of the buffer in bytes
+ * @return 0 on success, AVERROR_INVALIDDATA if the buffer_size would overflow.
+ */
+static inline int init_get_bits8(GetBitContext *s, const uint8_t *buffer,
+                                 int byte_size)
+{
+    if (byte_size > INT_MAX / 8)
+        return AVERROR_INVALIDDATA;
+    return init_get_bits(s, buffer, byte_size * 8);
 }
 
 static inline void align_get_bits(GetBitContext *s)
diff --git a/libavcodec/msrle.c b/libavcodec/msrle.c
index 5b7ba7fdad..5c1fb5c32a 100644
--- a/libavcodec/msrle.c
+++ b/libavcodec/msrle.c
@@ -67,7 +67,7 @@ static av_cold int msrle_decode_init(AVCodecContext *avctx)
         break;
     default:
         av_log(avctx, AV_LOG_ERROR, "unsupported bits per sample\n");
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
     avcodec_get_frame_defaults(&s->frame);
@@ -88,15 +88,16 @@ static int msrle_decode_frame(AVCodecContext *avctx,
     int buf_size = avpkt->size;
     MsrleContext *s = avctx->priv_data;
     int istride = FFALIGN(avctx->width*avctx->bits_per_coded_sample, 32) / 8;
+    int ret;
 
     s->buf = buf;
     s->size = buf_size;
 
     s->frame.reference = 3;
     s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE;
-    if (avctx->reget_buffer(avctx, &s->frame)) {
+    if ((ret = avctx->reget_buffer(avctx, &s->frame)) < 0) {
         av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
-        return -1;
+        return ret;
     }
 
     if (avctx->bits_per_coded_sample > 1 && avctx->bits_per_coded_sample <= 8) {
diff --git a/libavcodec/pcx.c b/libavcodec/pcx.c
index bac818c586..d60e5020c2 100644
--- a/libavcodec/pcx.c
+++ b/libavcodec/pcx.c
@@ -147,8 +147,8 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     if (p->data[0])
         avctx->release_buffer(avctx, p);
 
-    if (av_image_check_size(w, h, 0, avctx))
-        return AVERROR_INVALIDDATA;
+    if ((ret = av_image_check_size(w, h, 0, avctx)) < 0)
+        return ret;
     if (w != avctx->width || h != avctx->height)
         avcodec_set_dimensions(avctx, w, h);
     if ((ret = ff_get_buffer(avctx, p)) < 0) {
diff --git a/libavcodec/tmv.c b/libavcodec/tmv.c
index 2179dfbd6f..d198426356 100644
--- a/libavcodec/tmv.c
+++ b/libavcodec/tmv.c
@@ -48,20 +48,21 @@ static int tmv_decode_frame(AVCodecContext *avctx, void *data,
     unsigned char_cols = avctx->width >> 3;
     unsigned char_rows = avctx->height >> 3;
     unsigned x, y, fg, bg, c;
+    int ret;
 
     if (tmv->pic.data[0])
         avctx->release_buffer(avctx, &tmv->pic);
 
-    if (ff_get_buffer(avctx, &tmv->pic) < 0) {
+    if ((ret = ff_get_buffer(avctx, &tmv->pic)) < 0) {
         av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
-        return -1;
+        return ret;
     }
 
     if (avpkt->size < 2*char_rows*char_cols) {
         av_log(avctx, AV_LOG_ERROR,
                "Input buffer too small, truncated sample?\n");
         *got_frame = 0;
-        return -1;
+        return AVERROR_INVALIDDATA;
     }
 
     tmv->pic.pict_type = AV_PICTURE_TYPE_I;
diff --git a/libavfilter/x86/yadif.asm b/libavfilter/x86/yadif.asm
index 6dc8decdd6..a8f798771f 100644
--- a/libavfilter/x86/yadif.asm
+++ b/libavfilter/x86/yadif.asm
@@ -31,8 +31,8 @@ pw_1: times  8 dw 1
 SECTION .text
 
 %macro CHECK 2
-    movu      m2, [curq+mrefsq+%1]
-    movu      m3, [curq+prefsq+%2]
+    movu      m2, [curq+t1+%1]
+    movu      m3, [curq+t0+%2]
     mova      m4, m2
     mova      m5, m2
     pxor      m4, m3
@@ -97,8 +97,8 @@ SECTION .text
 %macro FILTER 3
 .loop%1:
     pxor         m7, m7
-    LOAD          0, [curq+mrefsq]
-    LOAD          1, [curq+prefsq]
+    LOAD          0, [curq+t1]
+    LOAD          1, [curq+t0]
     LOAD          2, [%2]
     LOAD          3, [%3]
     mova         m4, m3
@@ -109,8 +109,8 @@ SECTION .text
     mova   [rsp+32], m1
     psubw        m2, m4
     ABS1         m2, m4
-    LOAD          3, [prevq+mrefsq]
-    LOAD          4, [prevq+prefsq]
+    LOAD          3, [prevq+t1]
+    LOAD          4, [prevq+t0]
     psubw        m3, m0
     psubw        m4, m1
     ABS1         m3, m5
@@ -119,8 +119,8 @@ SECTION .text
     psrlw        m2, 1
     psrlw        m3, 1
     pmaxsw       m2, m3
-    LOAD          3, [nextq+mrefsq]
-    LOAD          4, [nextq+prefsq]
+    LOAD          3, [nextq+t1]
+    LOAD          4, [nextq+t0]
     psubw        m3, m0
     psubw        m4, m1
     ABS1         m3, m5
@@ -136,8 +136,8 @@ SECTION .text
     psrlw        m1, 1
     ABS1         m0, m2
 
-    movu         m2, [curq+mrefsq-1]
-    movu         m3, [curq+prefsq-1]
+    movu         m2, [curq+t1-1]
+    movu         m3, [curq+t0-1]
     mova         m4, m2
     psubusb      m2, m3
     psubusb      m3, m4
@@ -164,12 +164,12 @@ SECTION .text
     CHECK2
 
     mova         m6, [rsp+48]
-    cmp DWORD modem, 2
+    cmp   DWORD r8m, 2
     jge .end%1
-    LOAD          2, [%2+mrefsq*2]
-    LOAD          4, [%3+mrefsq*2]
-    LOAD          3, [%2+prefsq*2]
-    LOAD          5, [%3+prefsq*2]
+    LOAD          2, [%2+t1*2]
+    LOAD          4, [%3+t1*2]
+    LOAD          3, [%2+t0*2]
+    LOAD          5, [%3+t0*2]
     paddw        m2, m4
     paddw        m3, m5
     psrlw        m2, 1
@@ -208,19 +208,31 @@ SECTION .text
     add       prevq, mmsize/2
     add        curq, mmsize/2
     add       nextq, mmsize/2
-    sub          wd, mmsize/2
+    sub   DWORD r4m, mmsize/2
     jg .loop%1
 %endmacro
 
 %macro YADIF 0
-cglobal yadif_filter_line, 7, 7, 8, 16*5, dst, prev, cur, next, w, prefs, \
-                                          mrefs, parity, mode
-    test             wq, wq
+%if ARCH_X86_32
+cglobal yadif_filter_line, 4, 6, 8, 80, dst, prev, cur, next, w, prefs, \
+                                        mrefs, parity, mode
+%else
+cglobal yadif_filter_line, 4, 7, 8, 80, dst, prev, cur, next, w, prefs, \
+                                        mrefs, parity, mode
+%endif
+    cmp      DWORD wm, 0
     jle .ret
-    movsxdifnidn prefsq, prefsd
-    movsxdifnidn mrefsq, mrefsd
+%if ARCH_X86_32
+    mov            r4, r5mp
+    mov            r5, r6mp
+    DECLARE_REG_TMP 4,5
+%else
+    movsxd         r5, DWORD r5m
+    movsxd         r6, DWORD r6m
+    DECLARE_REG_TMP 5,6
+%endif
 
-    cmp   DWORD paritym, 0
+    cmp DWORD paritym, 0
     je .parity0
     FILTER 1, prevq, curq
     jmp .ret