diff options
| author | Ronald S. Bultje <rsbultje@gmail.com> | 2011-07-08 14:57:07 -0700 |
|---|---|---|
| committer | Michael Niedermayer <michaelni@gmx.at> | 2011-07-11 03:51:46 +0200 |
| commit | 93a10dd56121862fab13aa6676227673c534cc12 (patch) | |
| tree | a6a58a6510b73c59596531457a61da46410dad71 | |
| parent | 71a04bc19dfb990d3f7cce82b5516e88d2c371f4 (diff) | |
| download | ffmpeg-93a10dd56121862fab13aa6676227673c534cc12.tar.gz | |
swscale: fix overflow in 16-bit vertical scaling.
We operated on 31-bits, but with e.g. lanczos scaling, values can
add up to beyond 0x80000000, thus leading to output of zeroes. Drop
one bit of precision fixes this.
| -rw-r--r-- | libswscale/swscale.c | 18 |
1 files changed, 9 insertions, 9 deletions
diff --git a/libswscale/swscale.c b/libswscale/swscale.c index b9c619c727..b6455bb5e7 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -302,7 +302,7 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc, int dword= output_bits == 16; uint16_t *yDest = dest[0], *uDest = dest[1], *vDest = dest[2], *aDest = CONFIG_SWSCALE_ALPHA ? dest[3] : NULL; - int shift = 11 + 4*dword + 16 - output_bits; + int shift = 11 + 4*dword + 16 - output_bits - 1; #define output_pixel(pos, val) \ if (big_endian) { \ @@ -319,24 +319,24 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc, } \ } for (i = 0; i < dstW; i++) { - int val = 1 << (26-output_bits + 4*dword); + int val = 1 << (26-output_bits + 4*dword - 1); int j; for (j = 0; j < lumFilterSize; j++) - val += (dword ? lumSrc[j][i] : ((int16_t**)lumSrc)[j][i]) * lumFilter[j]; + val += ((dword ? lumSrc[j][i] : ((int16_t**)lumSrc)[j][i]) * lumFilter[j])>>1; output_pixel(&yDest[i], val); } if (uDest) { for (i = 0; i < chrDstW; i++) { - int u = 1 << (26-output_bits + 4*dword); - int v = 1 << (26-output_bits + 4*dword); + int u = 1 << (26-output_bits + 4*dword - 1); + int v = 1 << (26-output_bits + 4*dword - 1); int j; for (j = 0; j < chrFilterSize; j++) { - u += (dword ? chrUSrc[j][i] : ((int16_t**)chrUSrc)[j][i]) * chrFilter[j]; - v += (dword ? chrVSrc[j][i] : ((int16_t**)chrVSrc)[j][i]) * chrFilter[j]; + u += ((dword ? chrUSrc[j][i] : ((int16_t**)chrUSrc)[j][i]) * chrFilter[j]) >> 1; + v += ((dword ? chrVSrc[j][i] : ((int16_t**)chrVSrc)[j][i]) * chrFilter[j]) >> 1; } output_pixel(&uDest[i], u); @@ -346,11 +346,11 @@ yuv2yuvX16_c_template(const int16_t *lumFilter, const int32_t **lumSrc, if (CONFIG_SWSCALE_ALPHA && aDest) { for (i = 0; i < dstW; i++) { - int val = 1 << (26-output_bits + 4*dword); + int val = 1 << (26-output_bits + 4*dword - 1); int j; for (j = 0; j < lumFilterSize; j++) - val += (dword ? alpSrc[j][i] : ((int16_t**)alpSrc)[j][i]) * lumFilter[j]; + val += ((dword ? alpSrc[j][i] : ((int16_t**)alpSrc)[j][i]) * lumFilter[j]) >> 1; output_pixel(&aDest[i], val); } |