aboutsummaryrefslogtreecommitdiffstats
path: root/libswscale
diff options
context:
space:
mode:
authorRonald S. Bultje <rsbultje@gmail.com>2011-12-17 19:56:40 +0000
committerMans Rullgard <mans@mansr.com>2011-12-17 22:41:53 +0000
commit72dafea0fc0eb7230d7ebb0a7bc803e13b72aaad (patch)
treecee4a2212c3d21555a2c899c848191662dd18d0c /libswscale
parent86a10281c7f6204b3ac1253718971b1f8c106699 (diff)
downloadffmpeg-72dafea0fc0eb7230d7ebb0a7bc803e13b72aaad.tar.gz
swscale: fix overflow in gray16 vertical scaling.
This fixes the same overflow as in the RGB48/16-bit YUV scaling; some filters can overflow both negatively and positively (e.g. spline/lanczos), so we bias a signed integer so it's "half signed" and "half unsigned", and can cover overflows in both directions while maintaining full 31-bit depth. Signed-off-by: Mans Rullgard <mans@mansr.com>
Diffstat (limited to 'libswscale')
-rw-r--r--libswscale/swscale.c14
1 files changed, 6 insertions, 8 deletions
diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 3ea37104e8..c9dfc8df2f 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -386,8 +386,8 @@ yuv2gray16_X_c_template(SwsContext *c, const int16_t *lumFilter,
for (i = 0; i < (dstW >> 1); i++) {
int j;
- int Y1 = 1 << 14;
- int Y2 = 1 << 14;
+ int Y1 = (1 << 14) - 0x40000000;
+ int Y2 = (1 << 14) - 0x40000000;
for (j = 0; j < lumFilterSize; j++) {
Y1 += lumSrc[j][i * 2] * lumFilter[j];
@@ -395,12 +395,10 @@ yuv2gray16_X_c_template(SwsContext *c, const int16_t *lumFilter,
}
Y1 >>= 15;
Y2 >>= 15;
- if ((Y1 | Y2) & 0x10000) {
- Y1 = av_clip_uint16(Y1);
- Y2 = av_clip_uint16(Y2);
- }
- output_pixel(&dest[i * 2 + 0], Y1);
- output_pixel(&dest[i * 2 + 1], Y2);
+ Y1 = av_clip_int16(Y1);
+ Y2 = av_clip_int16(Y2);
+ output_pixel(&dest[i * 2 + 0], 0x8000 + Y1);
+ output_pixel(&dest[i * 2 + 1], 0x8000 + Y2);
}
}