aboutsummaryrefslogtreecommitdiffstats
path: root/libavcodec/mss2.c
diff options
context:
space:
mode:
authorMichael Niedermayer <michaelni@gmx.at>2012-11-12 10:57:06 +0100
committerMichael Niedermayer <michaelni@gmx.at>2012-11-12 10:57:06 +0100
commitda501ea857b13ea13749e372ccca9ae0804ff4d5 (patch)
tree1f43b7a2ba9a1ee63c1c38cc87983cbec20d8075 /libavcodec/mss2.c
parent3a04c18d899d278eea551c216e5117974063062b (diff)
parent802713c4e7b41bc2deed754d78649945c3442063 (diff)
downloadffmpeg-da501ea857b13ea13749e372ccca9ae0804ff4d5.tar.gz
Merge commit '802713c4e7b41bc2deed754d78649945c3442063'
* commit '802713c4e7b41bc2deed754d78649945c3442063': mss2: prevent potential uninitialized reads mss2: reindent after last commit mss2: fix handling of unmasked implicit WMV9 rectangles configure: add lavu dependency to lavr/lavfi .pc files x86inc: Set program_name outside of x86inc.asm Conflicts: configure Merged-by: Michael Niedermayer <michaelni@gmx.at>
Diffstat (limited to 'libavcodec/mss2.c')
-rw-r--r--libavcodec/mss2.c81
1 files changed, 42 insertions, 39 deletions
diff --git a/libavcodec/mss2.c b/libavcodec/mss2.c
index 10687f0441..caeb800c61 100644
--- a/libavcodec/mss2.c
+++ b/libavcodec/mss2.c
@@ -474,7 +474,7 @@ static int mss2_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
int keyframe, has_wmv9, has_mv, is_rle, is_555, ret;
Rectangle wmv9rects[MAX_WMV9_RECTANGLES], *r;
- int used_rects = 0, i, implicit_rect, av_uninit(wmv9_mask);
+ int used_rects = 0, i, implicit_rect = 0, av_uninit(wmv9_mask);
av_assert0(FF_INPUT_BUFFER_PADDING_SIZE >=
ARITH2_PADDING + (MIN_CACHE_BITS + 7) / 8);
@@ -650,25 +650,6 @@ static int mss2_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
return AVERROR_INVALIDDATA;
buf_size -= bytestream2_tell(&gB);
- } else if (is_rle) {
- init_get_bits(&gb, buf, buf_size * 8);
- if (ret = decode_rle(&gb, c->pal_pic, c->pal_stride,
- c->rgb_pic, c->rgb_stride, c->pal, keyframe,
- ctx->split_position, 0,
- avctx->width, avctx->height))
- return ret;
- align_get_bits(&gb);
-
- if (c->slice_split)
- if (ret = decode_rle(&gb, c->pal_pic, c->pal_stride,
- c->rgb_pic, c->rgb_stride, c->pal, keyframe,
- ctx->split_position, 1,
- avctx->width, avctx->height))
- return ret;
-
- align_get_bits(&gb);
- buf += get_bits_count(&gb) >> 3;
- buf_size -= get_bits_count(&gb) >> 3;
} else {
if (keyframe) {
c->corrupted = 0;
@@ -676,32 +657,54 @@ static int mss2_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
if (c->slice_split)
ff_mss12_slicecontext_reset(&ctx->sc[1]);
}
- else if (c->corrupted)
- return AVERROR_INVALIDDATA;
- bytestream2_init(&gB, buf, buf_size + ARITH2_PADDING);
- arith2_init(&acoder, &gB);
- c->keyframe = keyframe;
- if (c->corrupted = ff_mss12_decode_rect(&ctx->sc[0], &acoder, 0, 0,
- avctx->width,
- ctx->split_position))
- return AVERROR_INVALIDDATA;
+ if (is_rle) {
+ init_get_bits(&gb, buf, buf_size * 8);
+ if (ret = decode_rle(&gb, c->pal_pic, c->pal_stride,
+ c->rgb_pic, c->rgb_stride, c->pal, keyframe,
+ ctx->split_position, 0,
+ avctx->width, avctx->height))
+ return ret;
+ align_get_bits(&gb);
- buf += arith2_get_consumed_bytes(&acoder);
- buf_size -= arith2_get_consumed_bytes(&acoder);
- if (c->slice_split) {
- if (buf_size < 1)
+ if (c->slice_split)
+ if (ret = decode_rle(&gb, c->pal_pic, c->pal_stride,
+ c->rgb_pic, c->rgb_stride, c->pal, keyframe,
+ ctx->split_position, 1,
+ avctx->width, avctx->height))
+ return ret;
+
+ align_get_bits(&gb);
+ buf += get_bits_count(&gb) >> 3;
+ buf_size -= get_bits_count(&gb) >> 3;
+ } else if (!implicit_rect || wmv9_mask != -1) {
+ if (c->corrupted)
return AVERROR_INVALIDDATA;
bytestream2_init(&gB, buf, buf_size + ARITH2_PADDING);
arith2_init(&acoder, &gB);
- if (c->corrupted = ff_mss12_decode_rect(&ctx->sc[1], &acoder, 0,
- ctx->split_position,
+ c->keyframe = keyframe;
+ if (c->corrupted = ff_mss12_decode_rect(&ctx->sc[0], &acoder, 0, 0,
avctx->width,
- avctx->height - ctx->split_position))
+ ctx->split_position))
return AVERROR_INVALIDDATA;
buf += arith2_get_consumed_bytes(&acoder);
buf_size -= arith2_get_consumed_bytes(&acoder);
- }
+ if (c->slice_split) {
+ if (buf_size < 1)
+ return AVERROR_INVALIDDATA;
+ bytestream2_init(&gB, buf, buf_size + ARITH2_PADDING);
+ arith2_init(&acoder, &gB);
+ if (c->corrupted = ff_mss12_decode_rect(&ctx->sc[1], &acoder, 0,
+ ctx->split_position,
+ avctx->width,
+ avctx->height - ctx->split_position))
+ return AVERROR_INVALIDDATA;
+
+ buf += arith2_get_consumed_bytes(&acoder);
+ buf_size -= arith2_get_consumed_bytes(&acoder);
+ }
+ } else
+ memset(c->pal_pic, 0, c->pal_stride * avctx->height);
}
if (has_wmv9) {
@@ -836,8 +839,8 @@ static av_cold int mss2_decode_init(AVCodecContext *avctx)
if (ret = ff_mss12_decode_init(c, 1, &ctx->sc[0], &ctx->sc[1]))
return ret;
c->pal_stride = c->mask_stride;
- c->pal_pic = av_malloc(c->pal_stride * avctx->height);
- c->last_pal_pic = av_malloc(c->pal_stride * avctx->height);
+ c->pal_pic = av_mallocz(c->pal_stride * avctx->height);
+ c->last_pal_pic = av_mallocz(c->pal_stride * avctx->height);
if (!c->pal_pic || !c->last_pal_pic) {
mss2_decode_end(avctx);
return AVERROR(ENOMEM);