avcodec/omx: Do not pass negative value into av_malloc()

Fixes CID1396849

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit bd83c295fc1b7f8001e5d134b912af86cd62c3f2)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

1 files changed, 4 insertions, 1 deletions

diff --git a/libavcodec/omx.c b/libavcodec/omx.c
index 1b2ae0d997..375dd1969c 100644
--- a/libavcodec/omx.c
+++ b/libavcodec/omx.c
@@ -761,7 +761,10 @@ static int omx_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
             } else {
                 // If not, we need to allocate a new buffer with the right
                 // size and copy the input frame into it.
-                uint8_t *buf = av_malloc(av_image_get_buffer_size(avctx->pix_fmt, s->stride, s->plane_size, 1));
+                uint8_t *buf = NULL;
+                int image_buffer_size = av_image_get_buffer_size(avctx->pix_fmt, s->stride, s->plane_size, 1);
+                if (image_buffer_size >= 0)
+                    buf = av_malloc(image_buffer_size);
                 if (!buf) {
                     // Return the buffer to the queue so it's not lost
                     append_buffer(&s->input_mutex, &s->input_cond, &s->num_free_in_buffers, s->free_in_buffers, buffer);