indeo: Sanitize ff_ivi_init_planes fail paths

Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
(cherry picked from commit 28dda8a691f1c723a4a9365ab85f9625f1330096)

Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

3 files changed, 8 insertions, 2 deletions

diff --git a/libavcodec/indeo4.c b/libavcodec/indeo4.c
index 7955db1215..42b11305e4 100644
--- a/libavcodec/indeo4.c
+++ b/libavcodec/indeo4.c
@@ -210,6 +210,7 @@ static int decode_pic_hdr(IVI45DecContext *ctx, AVCodecContext *avctx)
     if (ivi_pic_config_cmp(&pic_conf, &ctx->pic_conf)) {
         if (ff_ivi_init_planes(ctx->planes, &pic_conf)) {
             av_log(avctx, AV_LOG_ERROR, "Couldn't reallocate color planes!\n");
+            ctx->pic_conf.luma_bands = 0;
             return AVERROR(ENOMEM);
         }
 
diff --git a/libavcodec/indeo5.c b/libavcodec/indeo5.c
index 9b42962331..b7cfc5f09f 100644
--- a/libavcodec/indeo5.c
+++ b/libavcodec/indeo5.c
@@ -113,7 +113,7 @@ static int decode_gop_header(IVI45DecContext *ctx, AVCodecContext *avctx)
     }
 
     /* check if picture layout was changed and reallocate buffers */
-    if (ivi_pic_config_cmp(&pic_conf, &ctx->pic_conf)) {
+    if (ivi_pic_config_cmp(&pic_conf, &ctx->pic_conf) || ctx->gop_invalid) {
         result = ff_ivi_init_planes(ctx->planes, &pic_conf);
         if (result < 0) {
             av_log(avctx, AV_LOG_ERROR, "Couldn't reallocate color planes!\n");
@@ -314,9 +314,9 @@ static int decode_pic_hdr(IVI45DecContext *ctx, AVCodecContext *avctx)
     ctx->frame_num = get_bits(&ctx->gb, 8);
 
     if (ctx->frame_type == FRAMETYPE_INTRA) {
-        ctx->gop_invalid = 1;
         if ((ret = decode_gop_header(ctx, avctx)) < 0) {
             av_log(avctx, AV_LOG_ERROR, "Invalid GOP header, skipping frames.\n");
+            ctx->gop_invalid = 1;
             return ret;
         }
         ctx->gop_invalid = 0;
diff --git a/libavcodec/ivi_common.c b/libavcodec/ivi_common.c
index 66f94c132b..80d35725be 100644
--- a/libavcodec/ivi_common.c
+++ b/libavcodec/ivi_common.c
@@ -244,6 +244,7 @@ static av_cold void ivi_free_buffers(IVIPlaneDesc *planes)
             av_freep(&planes[p].bands[b].tiles);
         }
         av_freep(&planes[p].bands);
+        planes[p].num_bands = 0;
     }
 }
 
@@ -256,6 +257,10 @@ av_cold int ff_ivi_init_planes(IVIPlaneDesc *planes, const IVIPicConfig *cfg)
 
     ivi_free_buffers(planes);
 
+    if (cfg->pic_width < 1 || cfg->pic_height < 1 ||
+        cfg->luma_bands < 1 || cfg->chroma_bands < 1)
+        return AVERROR_INVALIDDATA;
+
     /* fill in the descriptor of the luminance plane */
     planes[0].width     = cfg->pic_width;
     planes[0].height    = cfg->pic_height;