indeo: Bound-check before applying transform

Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
(cherry picked from commit dc79685195a45c9b8b17d7b93d118e0aefa45462)

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

4 files changed, 58 insertions, 18 deletions

diff --git a/libavcodec/indeo4.c b/libavcodec/indeo4.c
index 8d00e7fd99..7955db1215 100644
--- a/libavcodec/indeo4.c
+++ b/libavcodec/indeo4.c
@@ -347,6 +347,13 @@ static int decode_band_hdr(IVI45DecContext *ctx, IVIBandDesc *band,
             band->inv_transform = transforms[transform_id].inv_trans;
             band->dc_transform  = transforms[transform_id].dc_trans;
             band->is_2d_trans   = transforms[transform_id].is_2d_trans;
+            if (transform_id < 10)
+                band->transform_size = 8;
+            else
+                band->transform_size = 4;
+
+            if (band->blk_size != band->transform_size)
+                return AVERROR_INVALIDDATA;
 
             scan_indx = get_bits(&ctx->gb, 4);
             if (scan_indx == 15) {
diff --git a/libavcodec/indeo5.c b/libavcodec/indeo5.c
index dc5f6f09ad..3bdb23f5f1 100644
--- a/libavcodec/indeo5.c
+++ b/libavcodec/indeo5.c
@@ -147,39 +147,47 @@ static int decode_gop_header(IVI45DecContext *ctx, AVCodecContext *avctx)
             /* select transform function and scan pattern according to plane and band number */
             switch ((p << 2) + i) {
             case 0:
-                band->inv_transform = ff_ivi_inverse_slant_8x8;
-                band->dc_transform  = ff_ivi_dc_slant_2d;
-                band->scan          = ff_zigzag_direct;
+                band->inv_transform  = ff_ivi_inverse_slant_8x8;
+                band->dc_transform   = ff_ivi_dc_slant_2d;
+                band->scan           = ff_zigzag_direct;
+                band->transform_size = 8;
                 break;
 
             case 1:
-                band->inv_transform = ff_ivi_row_slant8;
-                band->dc_transform  = ff_ivi_dc_row_slant;
-                band->scan          = ff_ivi_vertical_scan_8x8;
+                band->inv_transform  = ff_ivi_row_slant8;
+                band->dc_transform   = ff_ivi_dc_row_slant;
+                band->scan           = ff_ivi_vertical_scan_8x8;
+                band->transform_size = 8;
                 break;
 
             case 2:
-                band->inv_transform = ff_ivi_col_slant8;
-                band->dc_transform  = ff_ivi_dc_col_slant;
-                band->scan          = ff_ivi_horizontal_scan_8x8;
+                band->inv_transform  = ff_ivi_col_slant8;
+                band->dc_transform   = ff_ivi_dc_col_slant;
+                band->scan           = ff_ivi_horizontal_scan_8x8;
+                band->transform_size = 8;
                 break;
 
             case 3:
-                band->inv_transform = ff_ivi_put_pixels_8x8;
-                band->dc_transform  = ff_ivi_put_dc_pixel_8x8;
-                band->scan          = ff_ivi_horizontal_scan_8x8;
+                band->inv_transform  = ff_ivi_put_pixels_8x8;
+                band->dc_transform   = ff_ivi_put_dc_pixel_8x8;
+                band->scan           = ff_ivi_horizontal_scan_8x8;
+                band->transform_size = 8;
                 break;
 
             case 4:
-                band->inv_transform = ff_ivi_inverse_slant_4x4;
-                band->dc_transform  = ff_ivi_dc_slant_2d;
-                band->scan          = ff_ivi_direct_scan_4x4;
+                band->inv_transform  = ff_ivi_inverse_slant_4x4;
+                band->dc_transform   = ff_ivi_dc_slant_2d;
+                band->scan           = ff_ivi_direct_scan_4x4;
+                band->transform_size = 4;
                 break;
             }
 
             band->is_2d_trans = band->inv_transform == ff_ivi_inverse_slant_8x8 ||
                                 band->inv_transform == ff_ivi_inverse_slant_4x4;
 
+            if (band->transform_size != band->blk_size)
+                return AVERROR_INVALIDDATA;
+
             /* select dequant matrix according to plane and band number */
             if (!p) {
                 quant_mat = (pic_conf.luma_bands > 1) ? i+1 : 0;
diff --git a/libavcodec/ivi_common.c b/libavcodec/ivi_common.c
index 815a5cb112..4901feb37c 100644
--- a/libavcodec/ivi_common.c
+++ b/libavcodec/ivi_common.c
@@ -407,6 +407,24 @@ static int ivi_dec_tile_data_size(GetBitContext *gb)
     return len;
 }
 
+static int ivi_dc_transform(IVIBandDesc *band, int *prev_dc, int buf_offs,
+                            int blk_size)
+{
+    int buf_size = band->pitch * band->aheight - buf_offs;
+    int min_size = (blk_size - 1) * band->pitch + blk_size;
+
+    if (!band->dc_transform)
+        return 0;
+
+
+    if (min_size > buf_size)
+        return AVERROR_INVALIDDATA;
+
+    band->dc_transform(prev_dc, band->buf + buf_offs,
+                       band->pitch, blk_size);
+
+    return 0;
+}
 
 static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band,
                                    ivi_mc_func mc, int mv_x, int mv_y,
@@ -424,6 +442,12 @@ static int ivi_decode_coded_blocks(GetBitContext *gb, IVIBandDesc *band,
     int num_coeffs = blk_size * blk_size;
     int col_mask   = blk_size - 1;
     int scan_pos   = -1;
+    int min_size   = band->pitch * (band->transform_size - 1) +
+                     band->transform_size;
+    int buf_size   = band->pitch * band->aheight - offs;
+
+    if (min_size > buf_size)
+        return AVERROR_INVALIDDATA;
 
     if (!band->scan) {
         av_log(avctx, AV_LOG_ERROR, "Scan pattern is not set.\n");
@@ -585,9 +609,9 @@ static int ivi_decode_blocks(GetBitContext *gb, IVIBandDesc *band,
                 /* for intra blocks apply the dc slant transform */
                 /* for inter - perform the motion compensation without delta */
                 if (is_intra) {
-                    if (band->dc_transform)
-                        band->dc_transform(&prev_dc, band->buf + buf_offs,
-                                           band->pitch, blk_size);
+                    ret = ivi_dc_transform(band, &prev_dc, buf_offs, blk_size);
+                    if (ret < 0)
+                        return ret;
                 } else {
                     ret = ivi_mc(mc_no_delta_func, band->buf, band->ref_buf,
                                  buf_offs, mv_x, mv_y, band->pitch, mc_type);
diff --git a/libavcodec/ivi_common.h b/libavcodec/ivi_common.h
index 8ac41a5554..fb253745ac 100644
--- a/libavcodec/ivi_common.h
+++ b/libavcodec/ivi_common.h
@@ -159,6 +159,7 @@ typedef struct IVIBandDesc {
     int             num_tiles;      ///< number of tiles in this band
     IVITile         *tiles;         ///< array of tile descriptors
     InvTransformPtr *inv_transform;
+    int             transform_size;
     DCTransformPtr  *dc_transform;
     int             is_2d_trans;    ///< 1 indicates that the two-dimensional inverse transform is used
     int32_t         checksum;       ///< for debug purposes