diff options
| author | Michael Niedermayer <michaelni@gmx.at> | 2011-12-19 15:48:47 +0100 |
|---|---|---|
| committer | Michael Niedermayer <michaelni@gmx.at> | 2012-01-03 19:49:56 +0100 |
| commit | e9a7ab20be424b2175fdd4a74e98046a1ff6e36c (patch) | |
| tree | 9282f31a92bf369557ff3d0c48a81ed91cb8ed30 | |
| parent | 0ab9072ee032df3c716a98cf3b4fef2000053289 (diff) | |
| download | ffmpeg-e9a7ab20be424b2175fdd4a74e98046a1ff6e36c.tar.gz | |
swscale: Fix several (real world) integer overflows.
Fixes Ticket795
Bug Found by: Oana Stratulat
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 90cad5edf64511721421a8ffa2b0e3024e3e68c0)
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
| -rw-r--r-- | libswscale/utils.c | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/libswscale/utils.c b/libswscale/utils.c index 3240b96030..294dd1a458 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -220,7 +220,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi } else if (flags&SWS_POINT) { // lame looking point sampling mode int i; - int xDstInSrc; + int64_t xDstInSrc; filterSize= 1; FF_ALLOC_OR_GOTO(NULL, filter, dstW*sizeof(*filter)*filterSize, fail); @@ -234,7 +234,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi } } else if ((xInc <= (1<<16) && (flags&SWS_AREA)) || (flags&SWS_FAST_BILINEAR)) { // bilinear upscale int i; - int xDstInSrc; + int64_t xDstInSrc; filterSize= 2; FF_ALLOC_OR_GOTO(NULL, filter, dstW*sizeof(*filter)*filterSize, fail); @@ -246,7 +246,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi (*filterPos)[i]= xx; //bilinear upscale / linear interpolate / area averaging for (j=0; j<filterSize; j++) { - int64_t coeff= fone - FFABS((xx<<16) - xDstInSrc)*(fone>>16); + int64_t coeff= fone - FFABS(((int64_t)xx<<16) - xDstInSrc)*(fone>>16); if (coeff<0) coeff=0; filter[i*filterSize + j]= coeff; xx++; @@ -254,7 +254,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi xDstInSrc+= xInc; } } else { - int xDstInSrc; + int64_t xDstInSrc; int sizeFactor; if (flags&SWS_BICUBIC) sizeFactor= 4; @@ -283,7 +283,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi int j; (*filterPos)[i]= xx; for (j=0; j<filterSize; j++) { - int64_t d= ((int64_t)FFABS((xx<<17) - xDstInSrc))<<13; + int64_t d= (FFABS(((int64_t)xx<<17) - xDstInSrc))<<13; double floatd; int64_t coeff; |