diff options
author | Brian Kim <bkkim@google.com> | 2020-07-13 10:09:38 -0700 |
---|---|---|
committer | James Almer <jamrial@gmail.com> | 2020-07-22 11:42:54 -0300 |
commit | fccbd1245f391c7ba455f72f7896a6e392d39dfa (patch) | |
tree | 5ecdccad2eae94272519082079da3e0c8710b97d /libavutil/frame.c | |
parent | 3a8e9271765a07e509655346ef88a28578cbff47 (diff) | |
download | ffmpeg-fccbd1245f391c7ba455f72f7896a6e392d39dfa.tar.gz |
libavutil/frame: avoid UB when getting plane sizes
This uses av_image_fill_plane_sizes instead of av_image_fill_pointers
when we are getting plane sizes to avoid UB from adding offsets to NULL.
Signed-off-by: Brian Kim <bkkim@google.com>
Signed-off-by: James Almer <jamrial@gmail.com>
Diffstat (limited to 'libavutil/frame.c')
-rw-r--r-- | libavutil/frame.c | 20 |
1 files changed, 16 insertions, 4 deletions
diff --git a/libavutil/frame.c b/libavutil/frame.c index 9884eae054..3ab1aa3242 100644 --- a/libavutil/frame.c +++ b/libavutil/frame.c @@ -212,8 +212,10 @@ void av_frame_free(AVFrame **frame) static int get_video_buffer(AVFrame *frame, int align) { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format); - int ret, i, padded_height; + int ret, i, padded_height, total_size; int plane_padding = FFMAX(16 + 16/*STRIDE_ALIGN*/, align); + ptrdiff_t linesizes[4]; + size_t sizes[4]; if (!desc) return AVERROR(EINVAL); @@ -238,12 +240,22 @@ static int get_video_buffer(AVFrame *frame, int align) frame->linesize[i] = FFALIGN(frame->linesize[i], align); } + for (i = 0; i < 4; i++) + linesizes[i] = frame->linesize[i]; + padded_height = FFALIGN(frame->height, 32); - if ((ret = av_image_fill_pointers(frame->data, frame->format, padded_height, - NULL, frame->linesize)) < 0) + if ((ret = av_image_fill_plane_sizes(sizes, frame->format, + padded_height, linesizes)) < 0) return ret; - frame->buf[0] = av_buffer_alloc(ret + 4*plane_padding); + total_size = 4*plane_padding; + for (i = 0; i < 4; i++) { + if (sizes[i] > INT_MAX - total_size) + return AVERROR(EINVAL); + total_size += sizes[i]; + } + + frame->buf[0] = av_buffer_alloc(total_size); if (!frame->buf[0]) { ret = AVERROR(ENOMEM); goto fail; |