diff options
| author | Brian Kim <bkkim@google.com> | 2020-07-13 10:09:39 -0700 |
|---|---|---|
| committer | James Almer <jamrial@gmail.com> | 2020-07-22 11:42:54 -0300 |
| commit | c40d36076ae695021505e8ca1a59a9be4009997c (patch) | |
| tree | c757579cb079a4c07fb925ac327f17726d74f83a /libavcodec | |
| parent | fccbd1245f391c7ba455f72f7896a6e392d39dfa (diff) | |
| download | ffmpeg-c40d36076ae695021505e8ca1a59a9be4009997c.tar.gz | |
libavcodec/decode: avoid UB when getting plane sizes
This uses av_image_fill_plane_sizes instead of av_image_fill_pointers
when we are getting plane sizes to avoid UB from adding offsets to NULL.
Signed-off-by: Brian Kim <bkkim@google.com>
Signed-off-by: James Almer <jamrial@gmail.com>
Diffstat (limited to 'libavcodec')
| -rw-r--r-- | libavcodec/decode.c | 23 |
1 files changed, 11 insertions, 12 deletions
diff --git a/libavcodec/decode.c b/libavcodec/decode.c index f2244fc9d9..da587ac1f6 100644 --- a/libavcodec/decode.c +++ b/libavcodec/decode.c @@ -1471,12 +1471,12 @@ static int update_frame_pool(AVCodecContext *avctx, AVFrame *frame) switch (avctx->codec_type) { case AVMEDIA_TYPE_VIDEO: { - uint8_t *data[4]; int linesize[4]; - int size[4] = { 0 }; int w = frame->width; int h = frame->height; - int tmpsize, unaligned; + int unaligned; + ptrdiff_t linesize1[4]; + size_t size[4]; avcodec_align_dimensions2(avctx, &w, &h, pool->stride_align); @@ -1494,20 +1494,19 @@ static int update_frame_pool(AVCodecContext *avctx, AVFrame *frame) unaligned |= linesize[i] % pool->stride_align[i]; } while (unaligned); - tmpsize = av_image_fill_pointers(data, avctx->pix_fmt, h, - NULL, linesize); - if (tmpsize < 0) { - ret = tmpsize; + for (i = 0; i < 4; i++) + linesize1[i] = linesize[i]; + ret = av_image_fill_plane_sizes(size, avctx->pix_fmt, h, linesize1); + if (ret < 0) goto fail; - } - - for (i = 0; i < 3 && data[i + 1]; i++) - size[i] = data[i + 1] - data[i]; - size[i] = tmpsize - (data[i] - data[0]); for (i = 0; i < 4; i++) { pool->linesize[i] = linesize[i]; if (size[i]) { + if (size[i] > INT_MAX - (16 + STRIDE_ALIGN - 1)) { + ret = AVERROR(EINVAL); + goto fail; + } pool->pools[i] = av_buffer_pool_init(size[i] + 16 + STRIDE_ALIGN - 1, CONFIG_MEMORY_POISONING ? NULL : |