avutil: Add av_image_check_size2()

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit f542b152aa2086b30d1089162d79f5c136905c0c)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

2 files changed, 38 insertions, 5 deletions

diff --git a/libavutil/imgutils.c b/libavutil/imgutils.c
index 37808e53d0..cc410abad1 100644
--- a/libavutil/imgutils.c
+++ b/libavutil/imgutils.c
@@ -248,19 +248,38 @@ static const AVClass imgutils_class = {
     .parent_log_context_offset = offsetof(ImgUtils, log_ctx),
 };
 
-int av_image_check_size(unsigned int w, unsigned int h, int log_offset, void *log_ctx)
+int av_image_check_size2(unsigned int w, unsigned int h, int64_t max_pixels, enum AVPixelFormat pix_fmt, int log_offset, void *log_ctx)
 {
     ImgUtils imgutils = {
         .class      = &imgutils_class,
         .log_offset = log_offset,
         .log_ctx    = log_ctx,
     };
+    int64_t stride = av_image_get_linesize(pix_fmt, w, 0);
+    if (stride <= 0)
+        stride = 8LL*w;
+    stride += 128*8;
 
-    if ((int)w>0 && (int)h>0 && (w+128)*(uint64_t)(h+128) < INT_MAX/8)
-        return 0;
+    if ((int)w<=0 || (int)h<=0 || stride >= INT_MAX || stride*(uint64_t)(h+128) >= INT_MAX) {
+        av_log(&imgutils, AV_LOG_ERROR, "Picture size %ux%u is invalid\n", w, h);
+        return AVERROR(EINVAL);
+    }
 
-    av_log(&imgutils, AV_LOG_ERROR, "Picture size %ux%u is invalid\n", w, h);
-    return AVERROR(EINVAL);
+    if (max_pixels < INT64_MAX) {
+        if (w*(int64_t)h > max_pixels) {
+            av_log(&imgutils, AV_LOG_ERROR,
+                    "Picture size %ux%u exceeds specified max pixel count %"PRId64", see the documentation if you wish to increase it\n",
+                    w, h, max_pixels);
+            return AVERROR(EINVAL);
+        }
+    }
+
+    return 0;
+}
+
+int av_image_check_size(unsigned int w, unsigned int h, int log_offset, void *log_ctx)
+{
+    return av_image_check_size2(w, h, INT64_MAX, AV_PIX_FMT_NONE, log_offset, log_ctx);
 }
 
 int av_image_check_sar(unsigned int w, unsigned int h, AVRational sar)
diff --git a/libavutil/imgutils.h b/libavutil/imgutils.h
index 23282a38fa..19f34deced 100644
--- a/libavutil/imgutils.h
+++ b/libavutil/imgutils.h
@@ -192,6 +192,20 @@ int av_image_copy_to_buffer(uint8_t *dst, int dst_size,
 int av_image_check_size(unsigned int w, unsigned int h, int log_offset, void *log_ctx);
 
 /**
+ * Check if the given dimension of an image is valid, meaning that all
+ * bytes of the image can be addressed with a signed int.
+ *
+ * @param w the width of the picture
+ * @param h the height of the picture
+ * @param max_pixels the maximum number of pixels the user wants to accept
+ * @param pix_fmt the pixel format, can be AV_PIX_FMT_NONE if unknown.
+ * @param log_offset the offset to sum to the log level for logging with log_ctx
+ * @param log_ctx the parent logging context, it may be NULL
+ * @return >= 0 if valid, a negative error code otherwise
+ */
+int av_image_check_size2(unsigned int w, unsigned int h, int64_t max_pixels, enum AVPixelFormat pix_fmt, int log_offset, void *log_ctx);
+
+/**
  * Check if the given sample aspect ratio of an image is valid.
  *
  * It is considered invalid if the denominator is 0 or if applying the ratio