Don't use ff_cropTbl[] for IDCT.

Results of IDCT can by far outreach the range of ff_cropTbl[], leading
to overreads and potentially crashes.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org

1 files changed, 33 insertions, 35 deletions

diff --git a/libavcodec/vp3dsp.c b/libavcodec/vp3dsp.c
index baa22a5519..438ae76b57 100644
--- a/libavcodec/vp3dsp.c
+++ b/libavcodec/vp3dsp.c
@@ -41,7 +41,6 @@
 static av_always_inline void idct(uint8_t *dst, int stride, int16_t *input, int type)
 {
     int16_t *ip = input;
-    uint8_t *cm = ff_cropTbl + MAX_NEG_CROP;
 
     int A, B, C, D, Ad, Bd, Cd, Dd, E, F, G, H;
     int Ed, Gd, Add, Bdd, Fd, Hd;
@@ -147,29 +146,29 @@ static av_always_inline void idct(uint8_t *dst, int stride, int16_t *input, int
                 ip[5*8] = (Fd + Bdd ) >> 4;
                 ip[6*8] = (Fd - Bdd ) >> 4;
             }else if(type==1){
-                dst[0*stride] = cm[(Gd + Cd )  >> 4];
-                dst[7*stride] = cm[(Gd - Cd )  >> 4];
+                dst[0*stride] = av_clip_uint8((Gd + Cd )  >> 4);
+                dst[7*stride] = av_clip_uint8((Gd - Cd )  >> 4);
 
-                dst[1*stride] = cm[(Add + Hd ) >> 4];
-                dst[2*stride] = cm[(Add - Hd ) >> 4];
+                dst[1*stride] = av_clip_uint8((Add + Hd ) >> 4);
+                dst[2*stride] = av_clip_uint8((Add - Hd ) >> 4);
 
-                dst[3*stride] = cm[(Ed + Dd )  >> 4];
-                dst[4*stride] = cm[(Ed - Dd )  >> 4];
+                dst[3*stride] = av_clip_uint8((Ed + Dd )  >> 4);
+                dst[4*stride] = av_clip_uint8((Ed - Dd )  >> 4);
 
-                dst[5*stride] = cm[(Fd + Bdd ) >> 4];
-                dst[6*stride] = cm[(Fd - Bdd ) >> 4];
+                dst[5*stride] = av_clip_uint8((Fd + Bdd ) >> 4);
+                dst[6*stride] = av_clip_uint8((Fd - Bdd ) >> 4);
             }else{
-                dst[0*stride] = cm[dst[0*stride] + ((Gd + Cd )  >> 4)];
-                dst[7*stride] = cm[dst[7*stride] + ((Gd - Cd )  >> 4)];
+                dst[0*stride] = av_clip_uint8(dst[0*stride] + ((Gd + Cd )  >> 4));
+                dst[7*stride] = av_clip_uint8(dst[7*stride] + ((Gd - Cd )  >> 4));
 
-                dst[1*stride] = cm[dst[1*stride] + ((Add + Hd ) >> 4)];
-                dst[2*stride] = cm[dst[2*stride] + ((Add - Hd ) >> 4)];
+                dst[1*stride] = av_clip_uint8(dst[1*stride] + ((Add + Hd ) >> 4));
+                dst[2*stride] = av_clip_uint8(dst[2*stride] + ((Add - Hd ) >> 4));
 
-                dst[3*stride] = cm[dst[3*stride] + ((Ed + Dd )  >> 4)];
-                dst[4*stride] = cm[dst[4*stride] + ((Ed - Dd )  >> 4)];
+                dst[3*stride] = av_clip_uint8(dst[3*stride] + ((Ed + Dd )  >> 4));
+                dst[4*stride] = av_clip_uint8(dst[4*stride] + ((Ed - Dd )  >> 4));
 
-                dst[5*stride] = cm[dst[5*stride] + ((Fd + Bdd ) >> 4)];
-                dst[6*stride] = cm[dst[6*stride] + ((Fd - Bdd ) >> 4)];
+                dst[5*stride] = av_clip_uint8(dst[5*stride] + ((Fd + Bdd ) >> 4));
+                dst[6*stride] = av_clip_uint8(dst[6*stride] + ((Fd - Bdd ) >> 4));
             }
 
         } else {
@@ -190,18 +189,18 @@ static av_always_inline void idct(uint8_t *dst, int stride, int16_t *input, int
                 dst[4*stride]=
                 dst[5*stride]=
                 dst[6*stride]=
-                dst[7*stride]= cm[128 + ((xC4S4 * ip[0*8] + (IdctAdjustBeforeShift<<16))>>20)];
+                dst[7*stride]= av_clip_uint8(128 + ((xC4S4 * ip[0*8] + (IdctAdjustBeforeShift<<16))>>20));
             }else{
                 if(ip[0*8]){
                     int v= ((xC4S4 * ip[0*8] + (IdctAdjustBeforeShift<<16))>>20);
-                    dst[0*stride] = cm[dst[0*stride] + v];
-                    dst[1*stride] = cm[dst[1*stride] + v];
-                    dst[2*stride] = cm[dst[2*stride] + v];
-                    dst[3*stride] = cm[dst[3*stride] + v];
-                    dst[4*stride] = cm[dst[4*stride] + v];
-                    dst[5*stride] = cm[dst[5*stride] + v];
-                    dst[6*stride] = cm[dst[6*stride] + v];
-                    dst[7*stride] = cm[dst[7*stride] + v];
+                    dst[0*stride] = av_clip_uint8(dst[0*stride] + v);
+                    dst[1*stride] = av_clip_uint8(dst[1*stride] + v);
+                    dst[2*stride] = av_clip_uint8(dst[2*stride] + v);
+                    dst[3*stride] = av_clip_uint8(dst[3*stride] + v);
+                    dst[4*stride] = av_clip_uint8(dst[4*stride] + v);
+                    dst[5*stride] = av_clip_uint8(dst[5*stride] + v);
+                    dst[6*stride] = av_clip_uint8(dst[6*stride] + v);
+                    dst[7*stride] = av_clip_uint8(dst[7*stride] + v);
                 }
             }
         }
@@ -225,17 +224,16 @@ void ff_vp3_idct_add_c(uint8_t *dest/*align 8*/, int line_size, DCTELEM *block/*
 
 void ff_vp3_idct_dc_add_c(uint8_t *dest/*align 8*/, int line_size, const DCTELEM *block/*align 16*/){
     int i, dc = (block[0] + 15) >> 5;
-    const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP + dc;
 
     for(i = 0; i < 8; i++){
-        dest[0] = cm[dest[0]];
-        dest[1] = cm[dest[1]];
-        dest[2] = cm[dest[2]];
-        dest[3] = cm[dest[3]];
-        dest[4] = cm[dest[4]];
-        dest[5] = cm[dest[5]];
-        dest[6] = cm[dest[6]];
-        dest[7] = cm[dest[7]];
+        dest[0] = av_clip_uint8(dest[0] + dc);
+        dest[1] = av_clip_uint8(dest[1] + dc);
+        dest[2] = av_clip_uint8(dest[2] + dc);
+        dest[3] = av_clip_uint8(dest[3] + dc);
+        dest[4] = av_clip_uint8(dest[4] + dc);
+        dest[5] = av_clip_uint8(dest[5] + dc);
+        dest[6] = av_clip_uint8(dest[6] + dc);
+        dest[7] = av_clip_uint8(dest[7] + dc);
         dest += line_size;
     }
 }