avcodec/flicvideo: fix decoding with AVFrame's negative linesize

1 files changed, 48 insertions, 32 deletions

diff --git a/libavcodec/flicvideo.c b/libavcodec/flicvideo.c
index 8531eb715c..f0d32bab38 100644
--- a/libavcodec/flicvideo.c
+++ b/libavcodec/flicvideo.c
@@ -60,12 +60,28 @@
 #define FLC_DTA_TYPE_CODE (0xAF44) /* Marks an "Extended FLC" comes from Dave's Targa Animator (DTA) */
 #define FLC_MAGIC_CARPET_SYNTHETIC_TYPE_CODE (0xAF13)
 
+static inline int check_pixel_ptr(ptrdiff_t ptr, int n,
+                                  ptrdiff_t limit, int direction)
+{
+    if (( direction && ptr + n > limit) ||
+        (!direction && ptr + n < limit))
+        return AVERROR_INVALIDDATA;
+    return 0;
+}
+
 #define CHECK_PIXEL_PTR(n) \
-    if (pixel_ptr + n > pixel_limit) { \
-        av_log (s->avctx, AV_LOG_ERROR, "Invalid pixel_ptr = %d > pixel_limit = %d\n", \
-        pixel_ptr + n, pixel_limit); \
-        return AVERROR_INVALIDDATA; \
-    } \
+{ \
+    ret = check_pixel_ptr(pixel_ptr, (n), pixel_limit, direction); \
+    if (ret < 0) \
+        return ret; \
+}
+
+#define CHECK_Y_PTR() \
+{ \
+    ret = check_pixel_ptr(y_ptr, 0, pixel_limit, direction); \
+    if (ret < 0) \
+        return ret; \
+}
 
 typedef struct FlicDecodeContext {
     AVCodecContext *avctx;
@@ -153,7 +169,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
     FlicDecodeContext *s = avctx->priv_data;
 
     GetByteContext g2;
-    int pixel_ptr;
+    ptrdiff_t pixel_ptr;
     int palette_ptr;
     unsigned char palette_idx1;
     unsigned char palette_idx2;
@@ -164,7 +180,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
     unsigned int chunk_size;
     int chunk_type;
 
-    int i, j, ret;
+    int i, j, ret, direction;
 
     int color_packets;
     int color_changes;
@@ -175,18 +191,19 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
     int compressed_lines;
     int starting_line;
     int line_packets;
-    int y_ptr;
+    ptrdiff_t y_ptr;
     int byte_run;
     int pixel_skip;
     int pixel_countdown;
     unsigned char *pixels;
-    unsigned int pixel_limit;
+    ptrdiff_t pixel_limit;
 
     bytestream2_init(&g2, buf, buf_size);
 
     if ((ret = ff_reget_buffer(avctx, s->frame, 0)) < 0)
         return ret;
 
+    direction = s->frame->linesize[0] > 0;
     pixels = s->frame->data[0];
     pixel_limit = s->avctx->height * s->frame->linesize[0];
     if (buf_size < 16 || buf_size > INT_MAX - (3 * 256 + AV_INPUT_BUFFER_PADDING_SIZE))
@@ -271,8 +288,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
             while (compressed_lines > 0) {
                 if (bytestream2_tell(&g2) + 2 > stream_ptr_after_chunk)
                     break;
-                if (y_ptr > pixel_limit)
-                    return AVERROR_INVALIDDATA;
+                CHECK_Y_PTR()
                 line_packets = sign_extend(bytestream2_get_le16(&g2), 16);
                 if ((line_packets & 0xC000) == 0xC000) {
                     // line skip opcode
@@ -374,8 +390,8 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
 
         case FLI_BLACK:
             /* set the whole frame to color 0 (which is usually black) */
-            memset(pixels, 0,
-                s->frame->linesize[0] * s->avctx->height);
+            for (int y = 0; y < s->avctx->height; y++)
+                memset(pixels + y * s->frame->linesize[0], 0, s->avctx->width);
             break;
 
         case FLI_BRUN:
@@ -433,7 +449,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
                        "has incorrect size, skipping chunk\n", chunk_size - 6);
                 bytestream2_skip(&g2, chunk_size - 6);
             } else {
-                for (y_ptr = 0; y_ptr < s->frame->linesize[0] * s->avctx->height;
+                for (y_ptr = 0; check_pixel_ptr(y_ptr, 0, pixel_limit, direction) == 0;
                      y_ptr += s->frame->linesize[0]) {
                     bytestream2_get_buffer(&g2, &pixels[y_ptr],
                                            s->avctx->width);
@@ -498,7 +514,7 @@ static int flic_decode_frame_15_16BPP(AVCodecContext *avctx,
     FlicDecodeContext *s = avctx->priv_data;
 
     GetByteContext g2;
-    int pixel_ptr;
+    ptrdiff_t pixel_ptr;
     unsigned char palette_idx1;
 
     unsigned int frame_size;
@@ -507,24 +523,25 @@ static int flic_decode_frame_15_16BPP(AVCodecContext *avctx,
     unsigned int chunk_size;
     int chunk_type;
 
-    int i, j, ret;
+    int i, j, ret, direction;
 
     int lines;
     int compressed_lines;
     int line_packets;
-    int y_ptr;
+    ptrdiff_t y_ptr;
     int byte_run;
     int pixel_skip;
     int pixel_countdown;
     unsigned char *pixels;
     int pixel;
-    unsigned int pixel_limit;
+    ptrdiff_t pixel_limit;
 
     bytestream2_init(&g2, buf, buf_size);
 
     if ((ret = ff_reget_buffer(avctx, s->frame, 0)) < 0)
         return ret;
 
+    direction = s->frame->linesize[0] > 0;
     pixels = s->frame->data[0];
     pixel_limit = s->avctx->height * s->frame->linesize[0];
 
@@ -573,8 +590,7 @@ static int flic_decode_frame_15_16BPP(AVCodecContext *avctx,
             while (compressed_lines > 0) {
                 if (bytestream2_tell(&g2) + 2 > stream_ptr_after_chunk)
                     break;
-                if (y_ptr > pixel_limit)
-                    return AVERROR_INVALIDDATA;
+                CHECK_Y_PTR()
                 line_packets = sign_extend(bytestream2_get_le16(&g2), 16);
                 if (line_packets < 0) {
                     line_packets = -line_packets;
@@ -625,8 +641,8 @@ static int flic_decode_frame_15_16BPP(AVCodecContext *avctx,
 
         case FLI_BLACK:
             /* set the whole frame to 0x0000 which is black in both 15Bpp and 16Bpp modes. */
-            memset(pixels, 0x0000,
-                   s->frame->linesize[0] * s->avctx->height);
+            for (int y = 0; y < s->avctx->height; y++)
+                memset(pixels + y * s->frame->linesize[0], 0, s->avctx->width * 2);
             break;
 
         case FLI_BRUN:
@@ -740,7 +756,7 @@ static int flic_decode_frame_15_16BPP(AVCodecContext *avctx,
 
                 if (bytestream2_get_bytes_left(&g2) < 2 * s->avctx->width * s->avctx->height )
                     return AVERROR_INVALIDDATA;
-                for (y_ptr = 0; y_ptr < s->frame->linesize[0] * s->avctx->height;
+                for (y_ptr = 0; check_pixel_ptr(y_ptr, 0, pixel_limit, direction) == 0;
                      y_ptr += s->frame->linesize[0]) {
 
                     pixel_countdown = s->avctx->width;
@@ -798,7 +814,7 @@ static int flic_decode_frame_24BPP(AVCodecContext *avctx,
     FlicDecodeContext *s = avctx->priv_data;
 
     GetByteContext g2;
-    int pixel_ptr;
+    ptrdiff_t pixel_ptr;
     unsigned char palette_idx1;
 
     unsigned int frame_size;
@@ -807,24 +823,25 @@ static int flic_decode_frame_24BPP(AVCodecContext *avctx,
     unsigned int chunk_size;
     int chunk_type;
 
-    int i, j, ret;
+    int i, j, ret, direction;
 
     int lines;
     int compressed_lines;
     int line_packets;
-    int y_ptr;
+    ptrdiff_t y_ptr;
     int byte_run;
     int pixel_skip;
     int pixel_countdown;
     unsigned char *pixels;
     int pixel;
-    unsigned int pixel_limit;
+    ptrdiff_t pixel_limit;
 
     bytestream2_init(&g2, buf, buf_size);
 
     if ((ret = ff_reget_buffer(avctx, s->frame, 0)) < 0)
         return ret;
 
+    direction = s->frame->linesize[0] > 0;
     pixels = s->frame->data[0];
     pixel_limit = s->avctx->height * s->frame->linesize[0];
 
@@ -873,8 +890,7 @@ static int flic_decode_frame_24BPP(AVCodecContext *avctx,
             while (compressed_lines > 0) {
                 if (bytestream2_tell(&g2) + 2 > stream_ptr_after_chunk)
                     break;
-                if (y_ptr > pixel_limit)
-                    return AVERROR_INVALIDDATA;
+                CHECK_Y_PTR()
                 line_packets = sign_extend(bytestream2_get_le16(&g2), 16);
                 if (line_packets < 0) {
                     line_packets = -line_packets;
@@ -926,8 +942,8 @@ static int flic_decode_frame_24BPP(AVCodecContext *avctx,
 
         case FLI_BLACK:
             /* set the whole frame to 0x00 which is black for 24 bit mode. */
-            memset(pixels, 0x00,
-                   s->frame->linesize[0] * s->avctx->height);
+            for (int y = 0; y < s->avctx->height; y++)
+                memset(pixels + y * s->frame->linesize[0], 0, s->avctx->width * 3);
             break;
 
         case FLI_BRUN:
@@ -1026,7 +1042,7 @@ static int flic_decode_frame_24BPP(AVCodecContext *avctx,
                        "bigger than image, skipping chunk\n", chunk_size - 6);
                 bytestream2_skip(&g2, chunk_size - 6);
             } else {
-                for (y_ptr = 0; y_ptr < s->frame->linesize[0] * s->avctx->height;
+                for (y_ptr = 0; check_pixel_ptr(y_ptr, 0, pixel_limit, direction) == 0;
                      y_ptr += s->frame->linesize[0]) {
 
                     bytestream2_get_buffer(&g2, pixels + y_ptr, 3*s->avctx->width);