aboutsummaryrefslogtreecommitdiffstats
path: root/libavutil/frame.c
diff options
context:
space:
mode:
authorBrian Kim <bkkim@google.com>2020-07-13 10:09:38 -0700
committerJames Almer <jamrial@gmail.com>2020-07-22 11:42:54 -0300
commitfccbd1245f391c7ba455f72f7896a6e392d39dfa (patch)
tree5ecdccad2eae94272519082079da3e0c8710b97d /libavutil/frame.c
parent3a8e9271765a07e509655346ef88a28578cbff47 (diff)
downloadffmpeg-fccbd1245f391c7ba455f72f7896a6e392d39dfa.tar.gz
libavutil/frame: avoid UB when getting plane sizes
This uses av_image_fill_plane_sizes instead of av_image_fill_pointers when we are getting plane sizes to avoid UB from adding offsets to NULL. Signed-off-by: Brian Kim <bkkim@google.com> Signed-off-by: James Almer <jamrial@gmail.com>
Diffstat (limited to 'libavutil/frame.c')
-rw-r--r--libavutil/frame.c20
1 files changed, 16 insertions, 4 deletions
diff --git a/libavutil/frame.c b/libavutil/frame.c
index 9884eae054..3ab1aa3242 100644
--- a/libavutil/frame.c
+++ b/libavutil/frame.c
@@ -212,8 +212,10 @@ void av_frame_free(AVFrame **frame)
static int get_video_buffer(AVFrame *frame, int align)
{
const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
- int ret, i, padded_height;
+ int ret, i, padded_height, total_size;
int plane_padding = FFMAX(16 + 16/*STRIDE_ALIGN*/, align);
+ ptrdiff_t linesizes[4];
+ size_t sizes[4];
if (!desc)
return AVERROR(EINVAL);
@@ -238,12 +240,22 @@ static int get_video_buffer(AVFrame *frame, int align)
frame->linesize[i] = FFALIGN(frame->linesize[i], align);
}
+ for (i = 0; i < 4; i++)
+ linesizes[i] = frame->linesize[i];
+
padded_height = FFALIGN(frame->height, 32);
- if ((ret = av_image_fill_pointers(frame->data, frame->format, padded_height,
- NULL, frame->linesize)) < 0)
+ if ((ret = av_image_fill_plane_sizes(sizes, frame->format,
+ padded_height, linesizes)) < 0)
return ret;
- frame->buf[0] = av_buffer_alloc(ret + 4*plane_padding);
+ total_size = 4*plane_padding;
+ for (i = 0; i < 4; i++) {
+ if (sizes[i] > INT_MAX - total_size)
+ return AVERROR(EINVAL);
+ total_size += sizes[i];
+ }
+
+ frame->buf[0] = av_buffer_alloc(total_size);
if (!frame->buf[0]) {
ret = AVERROR(ENOMEM);
goto fail;