Check pointers before writing to memory, fix possible integer overflows

Force alignement for mszh and zlib decoders

Originally committed as revision 3817 to svn://svn.ffmpeg.org/ffmpeg/trunk

1 files changed, 45 insertions, 16 deletions

diff --git a/libavcodec/lcl.c b/libavcodec/lcl.c
index 16225bb29b..cd2a3a8498 100644
--- a/libavcodec/lcl.c
+++ b/libavcodec/lcl.c
@@ -145,14 +145,15 @@ static inline unsigned char get_r (unsigned char yq, signed char rq)
 
 
 
-static int mszh_decomp(unsigned char * srcptr, int srclen, unsigned char * destptr)
+static unsigned int mszh_decomp(unsigned char * srcptr, int srclen, unsigned char * destptr, unsigned int destsize)
 {
     unsigned char *destptr_bak = destptr;
+    unsigned char *destptr_end = destptr + destsize;
     unsigned char mask = 0;
     unsigned char maskbit = 0;
     unsigned int ofs, cnt;
   
-    while (srclen > 0) {
+    while ((srclen > 0) && (destptr < destptr_end)) {
         if (maskbit == 0) {
             mask = *(srcptr++);
             maskbit = 8;
@@ -160,6 +161,8 @@ static int mszh_decomp(unsigned char * srcptr, int srclen, unsigned char * destp
             continue;
         }
         if ((mask & (1 << (--maskbit))) == 0) {
+            if (destptr + 4 > destptr_end)
+                break;
             *(int*)destptr = *(int*)srcptr;
             srclen -= 4;
             destptr += 4;
@@ -172,6 +175,9 @@ static int mszh_decomp(unsigned char * srcptr, int srclen, unsigned char * destp
             ofs &= 0x7ff;
             srclen -= 2;
             cnt *= 4;
+            if (destptr + cnt > destptr_end) {
+                cnt =  destptr_end - destptr;
+            }
             for (; cnt > 0; cnt--) {
                 *(destptr) = *(destptr - ofs);
                 destptr++;
@@ -194,7 +200,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8
 {
 	LclContext * const c = (LclContext *)avctx->priv_data;
 	unsigned char *encoded = (unsigned char *)buf;
-    int pixel_ptr;
+    unsigned int pixel_ptr;
     int row, col;
     unsigned char *outptr;
     unsigned int width = avctx->width; // Real image width
@@ -206,7 +212,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8
 #ifdef CONFIG_ZLIB
     int zret; // Zlib return code
 #endif
-    int len = buf_size;
+    unsigned int len = buf_size;
 
 	/* no supplementary picture */
 	if (buf_size == 0)
@@ -232,24 +238,29 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8
                     if (c->flags & FLAG_MULTITHREAD) {
                         mthread_inlen = *((unsigned int*)encoded);
                         mthread_outlen = *((unsigned int*)(encoded+4));
-                        mszh_dlen = mszh_decomp(encoded + 8, mthread_inlen, c->decomp_buf);
+                        if (mthread_outlen > c->decomp_size) // this should not happen
+                            mthread_outlen = c->decomp_size;
+                        mszh_dlen = mszh_decomp(encoded + 8, mthread_inlen, c->decomp_buf, c->decomp_size);
                         if (mthread_outlen != mszh_dlen) {
                             av_log(avctx, AV_LOG_ERROR, "Mthread1 decoded size differs (%d != %d)\n",
                                    mthread_outlen, mszh_dlen);
+                            return -1;
                         }
                         mszh_dlen = mszh_decomp(encoded + 8 + mthread_inlen, len - mthread_inlen,
-                                                c->decomp_buf + mthread_outlen);
-                        if ((c->decomp_size - mthread_outlen) != mszh_dlen) {
+                                                c->decomp_buf + mthread_outlen, c->decomp_size - mthread_outlen);
+                        if (mthread_outlen != mszh_dlen) {
                             av_log(avctx, AV_LOG_ERROR, "Mthread2 decoded size differs (%d != %d)\n",
-                                   c->decomp_size - mthread_outlen, mszh_dlen);
+                                   mthread_outlen, mszh_dlen);
+                            return -1;
                         }
                         encoded = c->decomp_buf;
                         len = c->decomp_size;
                     } else {
-                        mszh_dlen = mszh_decomp(encoded, len, c->decomp_buf);
+                        mszh_dlen = mszh_decomp(encoded, len, c->decomp_buf, c->decomp_size);
                         if (c->decomp_size != mszh_dlen) {
                             av_log(avctx, AV_LOG_ERROR, "Decoded size differs (%d != %d)\n",
                                    c->decomp_size, mszh_dlen);
+                            return -1;
                         }
                         encoded = c->decomp_buf;
                         len = mszh_dlen;
@@ -278,10 +289,12 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8
             if (c->flags & FLAG_MULTITHREAD) {
                 mthread_inlen = *((unsigned int*)encoded);
                 mthread_outlen = *((unsigned int*)(encoded+4));
+                if (mthread_outlen > c->decomp_size)
+                    mthread_outlen = c->decomp_size;
                 c->zstream.next_in = encoded + 8;
                 c->zstream.avail_in = mthread_inlen;
                 c->zstream.next_out = c->decomp_buf;
-                c->zstream.avail_out = mthread_outlen;    
+                c->zstream.avail_out = c->decomp_size;    
                 zret = inflate(&(c->zstream), Z_FINISH);
                 if ((zret != Z_OK) && (zret != Z_STREAM_END)) {
                     av_log(avctx, AV_LOG_ERROR, "Mthread1 inflate error: %d\n", zret);
@@ -290,6 +303,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8
                 if (mthread_outlen != (unsigned int)(c->zstream.total_out)) {
                     av_log(avctx, AV_LOG_ERROR, "Mthread1 decoded size differs (%u != %lu)\n",
                            mthread_outlen, c->zstream.total_out);
+                    return -1;
                 }
                 zret = inflateReset(&(c->zstream));
                 if (zret != Z_OK) {
@@ -299,15 +313,16 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8
                 c->zstream.next_in = encoded + 8 + mthread_inlen;
                 c->zstream.avail_in = len - mthread_inlen;
                 c->zstream.next_out = c->decomp_buf + mthread_outlen;
-                c->zstream.avail_out = mthread_outlen;    
+                c->zstream.avail_out = c->decomp_size - mthread_outlen;    
                 zret = inflate(&(c->zstream), Z_FINISH);
                 if ((zret != Z_OK) && (zret != Z_STREAM_END)) {
                     av_log(avctx, AV_LOG_ERROR, "Mthread2 inflate error: %d\n", zret);
                     return -1;
                 }
-                if ((c->decomp_size - mthread_outlen) != (unsigned int)(c->zstream.total_out)) {
+                if (mthread_outlen != (unsigned int)(c->zstream.total_out)) {
                     av_log(avctx, AV_LOG_ERROR, "Mthread2 decoded size differs (%d != %lu)\n",
-                           c->decomp_size - mthread_outlen, c->zstream.total_out);
+                           mthread_outlen, c->zstream.total_out);
+                    return -1;
                 }
             } else {
                 c->zstream.next_in = encoded;
@@ -322,6 +337,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, uint8
                 if (c->decomp_size != (unsigned int)(c->zstream.total_out)) {
                     av_log(avctx, AV_LOG_ERROR, "Decoded size differs (%d != %lu)\n",
                            c->decomp_size, c->zstream.total_out);
+                    return -1;
                 }
             }
             encoded = c->decomp_buf;
@@ -604,7 +620,9 @@ static int encode_frame(AVCodecContext *avctx, unsigned char *buf, int buf_size,
 static int decode_init(AVCodecContext *avctx)
 {
     LclContext * const c = (LclContext *)avctx->priv_data;
-    int basesize = avctx->width * avctx->height;
+    unsigned int basesize = avctx->width * avctx->height;
+    unsigned int max_basesize = ((avctx->width + 3) & ~3) * ((avctx->height + 3) & ~3);
+    unsigned int max_decomp_size;
     int zret; // Zlib return code
 
     c->avctx = avctx;
@@ -622,6 +640,12 @@ static int decode_init(AVCodecContext *avctx)
         return 1;
     }
 
+    // FIXME: find a better way to prevent integer overflow
+    if (((unsigned int)avctx->width > 32000) || ((unsigned int)avctx->height > 32000)) {
+        av_log(avctx, AV_LOG_ERROR, "Bad image size (w = %d, h = %d).\n", avctx->width, avctx->height);
+        return 1;
+    }
+
     /* Check codec type */ 
     if (((avctx->codec_id == CODEC_ID_MSZH)  && (*((char *)avctx->extradata + 7) != CODEC_MSZH)) ||
         ((avctx->codec_id == CODEC_ID_ZLIB)  && (*((char *)avctx->extradata + 7) != CODEC_ZLIB))) {
@@ -632,26 +656,32 @@ static int decode_init(AVCodecContext *avctx)
     switch (c->imgtype = *((char *)avctx->extradata + 4)) {
         case IMGTYPE_YUV111:
             c->decomp_size = basesize * 3;
+            max_decomp_size = max_basesize * 3;
             av_log(avctx, AV_LOG_INFO, "Image type is YUV 1:1:1.\n");
             break;
         case IMGTYPE_YUV422:
             c->decomp_size = basesize * 2;
+            max_decomp_size = max_basesize * 2;
             av_log(avctx, AV_LOG_INFO, "Image type is YUV 4:2:2.\n");
             break;
         case IMGTYPE_RGB24:
             c->decomp_size = basesize * 3;
+            max_decomp_size = max_basesize * 3;
             av_log(avctx, AV_LOG_INFO, "Image type is RGB 24.\n");
             break;
         case IMGTYPE_YUV411:
             c->decomp_size = basesize / 2 * 3;
+            max_decomp_size = max_basesize / 2 * 3;
             av_log(avctx, AV_LOG_INFO, "Image type is YUV 4:1:1.\n");
             break;
         case IMGTYPE_YUV211:
             c->decomp_size = basesize * 2;
+            max_decomp_size = max_basesize * 2;
             av_log(avctx, AV_LOG_INFO, "Image type is YUV 2:1:1.\n");
             break;
         case IMGTYPE_YUV420:
             c->decomp_size = basesize / 2 * 3;
+            max_decomp_size = max_basesize / 2 * 3;
             av_log(avctx, AV_LOG_INFO, "Image type is YUV 4:2:0.\n");
             break;
         default:
@@ -706,9 +736,8 @@ static int decode_init(AVCodecContext *avctx)
     }
 
     /* Allocate decompression buffer */
-    /* 4*8 max overflow space for mszh decomp algorithm */
     if (c->decomp_size) {
-        if ((c->decomp_buf = av_malloc(c->decomp_size+4*8)) == NULL) {
+        if ((c->decomp_buf = av_malloc(max_decomp_size)) == NULL) {
             av_log(avctx, AV_LOG_ERROR, "Can't allocate decompression buffer.\n");
             return 1;
         }