aboutsummaryrefslogtreecommitdiffstats
path: root/libswscale
diff options
context:
space:
mode:
authorRonald S. Bultje <rsbultje@gmail.com>2011-12-16 00:03:54 +0000
committerMans Rullgard <mans@mansr.com>2011-12-17 18:36:20 +0000
commitbe1bafc303a77fa9ca5a99e36a65d5765012d3f4 (patch)
tree6e7daa2b038cb79f72f8d0fc4ef04756eebad93d /libswscale
parent5d8122db5c0b537c4d2c3352b4c89cb92f865bc2 (diff)
downloadffmpeg-be1bafc303a77fa9ca5a99e36a65d5765012d3f4.tar.gz
swscale: fix overflows in output of RGB48 pixels.
For certain types of filters where the intermediate sum of coefficients can go above the fixed-point equivalent of 1.0 in the middle of a filter, the sum of a 31-bit calculation can overflow in both directions and can thus not be represented in a 32-bit signed or unsigned integer. To work around this, we subtract 0x40000000 from a signed integer base, so that we're halfway signed/unsigned, which makes it fit even if it overflows. After the filter finishes, we add the scaled bias back after a shift. We use the same trick for 16-bit bpc YUV output routines. Signed-off-by: Mans Rullgard <mans@mansr.com>
Diffstat (limited to 'libswscale')
-rw-r--r--libswscale/swscale.c6
1 files changed, 4 insertions, 2 deletions
diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index f072378001..818c049f35 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -754,8 +754,8 @@ yuv2rgb48_X_c_template(SwsContext *c, const int16_t *lumFilter,
for (i = 0; i < (dstW >> 1); i++) {
int j;
- int Y1 = 0;
- int Y2 = 0;
+ int Y1 = -0x40000000;
+ int Y2 = -0x40000000;
int U = -128 << 23; // 19
int V = -128 << 23;
int R, G, B;
@@ -771,7 +771,9 @@ yuv2rgb48_X_c_template(SwsContext *c, const int16_t *lumFilter,
// 8bit: 12+15=27; 16-bit: 12+19=31
Y1 >>= 14; // 10
+ Y1 += 0x10000;
Y2 >>= 14;
+ Y2 += 0x10000;
U >>= 14;
V >>= 14;