swscale/output: Fix all bilinear integer overflows

Ticket11686 hinted at one of these overflows
this fixes them all

Issue in line 1325/1326 found by HAORAN FANG <xfanghaoran@gmail.com>

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

1 files changed, 27 insertions, 22 deletions

diff --git a/libswscale/output.c b/libswscale/output.c
index b1025789aa..b873276c00 100644
--- a/libswscale/output.c
+++ b/libswscale/output.c
@@ -1053,27 +1053,28 @@ static av_always_inline void
 yuv2ya16_2_c_template(SwsInternal *c, const int32_t *buf[2],
                         const int32_t *unused_ubuf[2], const int32_t *unused_vbuf[2],
                         const int32_t *abuf[2], uint16_t *dest, int dstW,
-                        int yalpha, int unused_uvalpha, int y,
+                        int yalpha_param, int unused_uvalpha, int y,
                         enum AVPixelFormat target, int unused_hasAlpha,
                         int unused_eightbytes, int is_be)
 {
+    unsigned  yalpha =  yalpha_param;
     int hasAlpha = abuf && abuf[0] && abuf[1];
     const int32_t *buf0  = buf[0],  *buf1  = buf[1],
     *abuf0 = hasAlpha ? abuf[0] : NULL,
     *abuf1 = hasAlpha ? abuf[1] : NULL;
-    int  yalpha1 = 4096 - yalpha;
+    unsigned  yalpha1 = 4096 - yalpha;
     int i;
 
     av_assert2(yalpha  <= 4096U);
 
     for (i = 0; i < dstW; i++) {
-        int Y = (buf0[i] * yalpha1 + buf1[i] * yalpha) >> 15;
+        int Y = (int)(buf0[i] * yalpha1 + buf1[i] * yalpha) >> 15;
         int A;
 
         Y = av_clip_uint16(Y);
 
         if (hasAlpha) {
-            A = (abuf0[i] * yalpha1 + abuf1[i] * yalpha) >> 15;
+            A = (int)(abuf0[i] * yalpha1 + abuf1[i] * yalpha) >> 15;
             A = av_clip_uint16(A);
         }
 
@@ -1196,17 +1197,19 @@ static av_always_inline void
 yuv2rgba64_2_c_template(SwsInternal *c, const int32_t *buf[2],
                        const int32_t *ubuf[2], const int32_t *vbuf[2],
                        const int32_t *abuf[2], uint16_t *dest, int dstW,
-                       int yalpha, int uvalpha, int y,
+                       int yalpha_param, int uvalpha_param, int y,
                        enum AVPixelFormat target, int hasAlpha, int eightbytes,
                        int is_be)
 {
+    unsigned  yalpha =  yalpha_param;
+    unsigned uvalpha = uvalpha_param;
     const int32_t *buf0  = buf[0],  *buf1  = buf[1],
                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1],
                   *vbuf0 = vbuf[0], *vbuf1 = vbuf[1],
                   *abuf0 = hasAlpha ? abuf[0] : NULL,
                   *abuf1 = hasAlpha ? abuf[1] : NULL;
-    int  yalpha1 = 4096 - yalpha;
-    int uvalpha1 = 4096 - uvalpha;
+    unsigned  yalpha1 = 4096 - yalpha;
+    unsigned uvalpha1 = 4096 - uvalpha;
     int i;
     int A1 = 0xffff<<14, A2 = 0xffff<<14;
 
@@ -1214,10 +1217,10 @@ yuv2rgba64_2_c_template(SwsInternal *c, const int32_t *buf[2],
     av_assert2(uvalpha <= 4096U);
 
     for (i = 0; i < ((dstW + 1) >> 1); i++) {
-        unsigned Y1 = (buf0[i * 2]     * yalpha1  + buf1[i * 2]     * yalpha) >> 14;
-        unsigned Y2 = (buf0[i * 2 + 1] * yalpha1  + buf1[i * 2 + 1] * yalpha) >> 14;
-        int U  = (ubuf0[i]        * uvalpha1 + ubuf1[i]        * uvalpha - (128 << 23)) >> 14;
-        int V  = (vbuf0[i]        * uvalpha1 + vbuf1[i]        * uvalpha - (128 << 23)) >> 14;
+        unsigned Y1 = (int)(buf0[i * 2]     * yalpha1  + buf1[i * 2]     * yalpha) >> 14;
+        unsigned Y2 = (int)(buf0[i * 2 + 1] * yalpha1  + buf1[i * 2 + 1] * yalpha) >> 14;
+        int U  = (int)(ubuf0[i]        * uvalpha1 + ubuf1[i]        * uvalpha - (128 << 23)) >> 14;
+        int V  = (int)(vbuf0[i]        * uvalpha1 + vbuf1[i]        * uvalpha - (128 << 23)) >> 14;
         int R, G, B;
 
         Y1 -= c->yuv2rgb_y_offset;
@@ -1232,8 +1235,8 @@ yuv2rgba64_2_c_template(SwsInternal *c, const int32_t *buf[2],
         B =                            U * c->yuv2rgb_u2b_coeff;
 
         if (hasAlpha) {
-            A1 = (abuf0[i * 2    ] * yalpha1 + abuf1[i * 2    ] * yalpha) >> 1;
-            A2 = (abuf0[i * 2 + 1] * yalpha1 + abuf1[i * 2 + 1] * yalpha) >> 1;
+            A1 = (int)(abuf0[i * 2    ] * yalpha1 + abuf1[i * 2    ] * yalpha) >> 1;
+            A2 = (int)(abuf0[i * 2 + 1] * yalpha1 + abuf1[i * 2 + 1] * yalpha) >> 1;
 
             A1 += 1 << 13;
             A2 += 1 << 13;
@@ -1316,7 +1319,7 @@ yuv2rgba64_1_c_template(SwsInternal *c, const int32_t *buf0,
     } else {
         const int32_t *ubuf1 = ubuf[1], *vbuf1 = vbuf[1];
         int A1 = 0xffff<<14, A2 = 0xffff<<14;
-        int uvalpha1 = 4096 - uvalpha;
+        unsigned uvalpha1 = 4096 - uvalpha;
         av_assert2(uvalpha <= 4096U);
 
         for (i = 0; i < ((dstW + 1) >> 1); i++) {
@@ -1434,17 +1437,19 @@ static av_always_inline void
 yuv2rgba64_full_2_c_template(SwsInternal *c, const int32_t *buf[2],
                        const int32_t *ubuf[2], const int32_t *vbuf[2],
                        const int32_t *abuf[2], uint16_t *dest, int dstW,
-                       int yalpha, int uvalpha, int y,
+                       int yalpha_param, int uvalpha_param, int y,
                        enum AVPixelFormat target, int hasAlpha, int eightbytes,
                        int is_be)
 {
+    unsigned  yalpha =  yalpha_param;
+    unsigned uvalpha = uvalpha_param;
     const int32_t *buf0  = buf[0],  *buf1  = buf[1],
                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1],
                   *vbuf0 = vbuf[0], *vbuf1 = vbuf[1],
                   *abuf0 = hasAlpha ? abuf[0] : NULL,
                   *abuf1 = hasAlpha ? abuf[1] : NULL;
-    int  yalpha1 = 4096 - yalpha;
-    int uvalpha1 = 4096 - uvalpha;
+    unsigned  yalpha1 = 4096 - yalpha;
+    unsigned uvalpha1 = 4096 - uvalpha;
     int i;
     int A = 0xffff<<14;
 
@@ -1452,9 +1457,9 @@ yuv2rgba64_full_2_c_template(SwsInternal *c, const int32_t *buf[2],
     av_assert2(uvalpha <= 4096U);
 
     for (i = 0; i < dstW; i++) {
-        int Y  = (buf0[i]     * yalpha1  + buf1[i]     * yalpha) >> 14;
-        int U  = (ubuf0[i]   * uvalpha1 + ubuf1[i]     * uvalpha - (128 << 23)) >> 14;
-        int V  = (vbuf0[i]   * uvalpha1 + vbuf1[i]     * uvalpha - (128 << 23)) >> 14;
+        int Y  = (int)(buf0[i]     * yalpha1  + buf1[i]     * yalpha) >> 14;
+        int U  = (int)(ubuf0[i]   * uvalpha1 + ubuf1[i]     * uvalpha - (128 << 23)) >> 14;
+        int V  = (int)(vbuf0[i]   * uvalpha1 + vbuf1[i]     * uvalpha - (128 << 23)) >> 14;
         int R, G, B;
 
         Y -= c->yuv2rgb_y_offset;
@@ -1466,7 +1471,7 @@ yuv2rgba64_full_2_c_template(SwsInternal *c, const int32_t *buf[2],
         B =                            U * c->yuv2rgb_u2b_coeff;
 
         if (hasAlpha) {
-            A = (abuf0[i] * yalpha1 + abuf1[i] * yalpha) >> 1;
+            A = (int)(abuf0[i] * yalpha1 + abuf1[i] * yalpha) >> 1;
 
             A += 1 << 13;
         }
@@ -1527,7 +1532,7 @@ yuv2rgba64_full_1_c_template(SwsInternal *c, const int32_t *buf0,
         }
     } else {
         const int32_t *ubuf1 = ubuf[1], *vbuf1 = vbuf[1];
-        int uvalpha1 = 4096 - uvalpha;
+        unsigned uvalpha1 = 4096 - uvalpha;
         int A = 0xffff<<14;
         av_assert2(uvalpha <= 4096U);