Merge commit '7b1fbd4729a52dd7c02622dbe7bb81a6a7ed12f8'

* commit '7b1fbd4729a52dd7c02622dbe7bb81a6a7ed12f8':
  indeo2: check decoding errors.
  indeo2: return meaningful error codes
  rl2: cosmetics, reformat

Conflicts:
	libavcodec/rl2.c

Merged-by: Michael Niedermayer <michaelni@gmx.at>

2 files changed, 74 insertions, 59 deletions

diff --git a/libavcodec/indeo2.c b/libavcodec/indeo2.c
index fcd0ae46d4..8c4be4fb5e 100644
--- a/libavcodec/indeo2.c
+++ b/libavcodec/indeo2.c
@@ -57,7 +57,7 @@ static int ir2_decode_plane(Ir2Context *ctx, int width, int height, uint8_t *dst
     int t;
 
     if(width&1)
-        return -1;
+        return AVERROR_INVALIDDATA;
 
     /* first line contain absolute values, other lines contain deltas */
     while (out < width){
@@ -65,7 +65,7 @@ static int ir2_decode_plane(Ir2Context *ctx, int width, int height, uint8_t *dst
         if(c >= 0x80) { /* we have a run */
             c -= 0x7F;
             if(out + c*2 > width)
-                return -1;
+                return AVERROR_INVALIDDATA;
             for (i = 0; i < c * 2; i++)
                 dst[out++] = 0x80;
         } else { /* copy two values from table */
@@ -82,7 +82,7 @@ static int ir2_decode_plane(Ir2Context *ctx, int width, int height, uint8_t *dst
             if(c >= 0x80) { /* we have a skip */
                 c -= 0x7F;
                 if(out + c*2 > width)
-                    return -1;
+                    return AVERROR_INVALIDDATA;
                 for (i = 0; i < c * 2; i++) {
                     dst[out] = dst[out - stride];
                     out++;
@@ -112,7 +112,7 @@ static int ir2_decode_plane_inter(Ir2Context *ctx, int width, int height, uint8_
     int t;
 
     if(width&1)
-        return -1;
+        return AVERROR_INVALIDDATA;
 
     for (j = 0; j < height; j++){
         out = 0;
@@ -146,13 +146,13 @@ static int ir2_decode_frame(AVCodecContext *avctx,
     Ir2Context * const s = avctx->priv_data;
     AVFrame *picture = data;
     AVFrame * const p = &s->picture;
-    int start;
+    int start, ret;
 
     p->reference = 3;
     p->buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE;
-    if (avctx->reget_buffer(avctx, p)) {
+    if ((ret = avctx->reget_buffer(avctx, p)) < 0) {
         av_log(s->avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
-        return -1;
+        return ret;
     }
 
     start = 48; /* hardcoded for now */
@@ -173,21 +173,34 @@ static int ir2_decode_frame(AVCodecContext *avctx,
     init_get_bits(&s->gb, buf + start, (buf_size - start) * 8);
 
     if (s->decode_delta) { /* intraframe */
-        ir2_decode_plane(s, avctx->width, avctx->height,
-                         s->picture.data[0], s->picture.linesize[0], ir2_luma_table);
+        if ((ret = ir2_decode_plane(s, avctx->width, avctx->height,
+                                    s->picture.data[0], s->picture.linesize[0],
+                                    ir2_luma_table)) < 0)
+            return ret;
+
         /* swapped U and V */
-        ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2,
-                         s->picture.data[2], s->picture.linesize[2], ir2_luma_table);
-        ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2,
-                         s->picture.data[1], s->picture.linesize[1], ir2_luma_table);
+        if ((ret = ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2,
+                                    s->picture.data[2], s->picture.linesize[2],
+                                    ir2_luma_table)) < 0)
+            return ret;
+        if ((ret = ir2_decode_plane(s, avctx->width >> 2, avctx->height >> 2,
+                                    s->picture.data[1], s->picture.linesize[1],
+                                    ir2_luma_table)) < 0)
+            return ret;
     } else { /* interframe */
-        ir2_decode_plane_inter(s, avctx->width, avctx->height,
-                         s->picture.data[0], s->picture.linesize[0], ir2_luma_table);
+        if ((ret = ir2_decode_plane_inter(s, avctx->width, avctx->height,
+                                          s->picture.data[0], s->picture.linesize[0],
+                                          ir2_luma_table)) < 0)
+            return ret;
         /* swapped U and V */
-        ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2,
-                         s->picture.data[2], s->picture.linesize[2], ir2_luma_table);
-        ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2,
-                         s->picture.data[1], s->picture.linesize[1], ir2_luma_table);
+        if ((ret = ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2,
+                                          s->picture.data[2], s->picture.linesize[2],
+                                          ir2_luma_table)) < 0)
+            return ret;
+        if ((ret = ir2_decode_plane_inter(s, avctx->width >> 2, avctx->height >> 2,
+                                          s->picture.data[1], s->picture.linesize[1],
+                                          ir2_luma_table)) < 0)
+            return ret;
     }
 
     *picture   = s->picture;
diff --git a/libavcodec/rl2.c b/libavcodec/rl2.c
index a7115c555b..b908a83f48 100644
--- a/libavcodec/rl2.c
+++ b/libavcodec/rl2.c
@@ -66,60 +66,60 @@ static void rl2_rle_decode(Rl2Context *s, const uint8_t *in, int size,
     int stride_adj = stride - s->avctx->width;
     int i;
     const uint8_t *back_frame = s->back_frame;
-    const uint8_t *in_end = in + size;
-    const uint8_t *out_end = out + stride * s->avctx->height;
+    const uint8_t *in_end     = in + size;
+    const uint8_t *out_end    = out + stride * s->avctx->height;
     uint8_t *line_end;
 
     /** copy start of the background frame */
-    for(i=0;i<=base_y;i++){
-        if(s->back_frame)
-            memcpy(out,back_frame,s->avctx->width);
-        out += stride;
+    for (i = 0; i <= base_y; i++) {
+        if (s->back_frame)
+            memcpy(out, back_frame, s->avctx->width);
+        out        += stride;
         back_frame += s->avctx->width;
     }
     back_frame += base_x - s->avctx->width;
-    line_end = out - stride_adj;
-    out += base_x - stride;
+    line_end    = out - stride_adj;
+    out        += base_x - stride;
 
     /** decode the variable part of the frame */
-    while(in < in_end){
+    while (in < in_end) {
         uint8_t val = *in++;
-        int len = 1;
-        if(val >= 0x80){
-            if(in >= in_end)
+        int len     = 1;
+        if (val >= 0x80) {
+            if (in >= in_end)
                 break;
             len = *in++;
-            if(!len)
+            if (!len)
                 break;
         }
 
-        if(len >= out_end - out)
+        if (len >= out_end - out)
             break;
 
-        if(s->back_frame)
+        if (s->back_frame)
             val |= 0x80;
         else
             val &= ~0x80;
 
-        while(len--){
-            *out++ = (val == 0x80)? *back_frame:val;
+        while (len--) {
+            *out++ = (val == 0x80) ? *back_frame : val;
             back_frame++;
-            if(out == line_end){
-                 out += stride_adj;
+            if (out == line_end) {
+                 out      += stride_adj;
                  line_end += stride;
-                 if(len >= out_end - out)
+                 if (len >= out_end - out)
                      break;
             }
         }
     }
 
     /** copy the rest from the background frame */
-    if(s->back_frame){
-        while(out < out_end){
+    if (s->back_frame) {
+        while (out < out_end) {
             memcpy(out, back_frame, line_end - out);
             back_frame += line_end - out;
-            out = line_end + stride_adj;
-            line_end += stride;
+            out         = line_end + stride_adj;
+            line_end   += stride;
         }
     }
 }
@@ -135,38 +135,39 @@ static av_cold int rl2_decode_init(AVCodecContext *avctx)
     Rl2Context *s = avctx->priv_data;
     int back_size;
     int i;
-    s->avctx = avctx;
+
+    s->avctx       = avctx;
     avctx->pix_fmt = AV_PIX_FMT_PAL8;
     avcodec_get_frame_defaults(&s->frame);
 
     /** parse extra data */
-    if(!avctx->extradata || avctx->extradata_size < EXTRADATA1_SIZE){
+    if (!avctx->extradata || avctx->extradata_size < EXTRADATA1_SIZE) {
         av_log(avctx, AV_LOG_ERROR, "invalid extradata size\n");
         return AVERROR(EINVAL);
     }
 
     /** get frame_offset */
     s->video_base = AV_RL16(&avctx->extradata[0]);
-    s->clr_count = AV_RL32(&avctx->extradata[2]);
+    s->clr_count  = AV_RL32(&avctx->extradata[2]);
 
-    if(s->video_base >= avctx->width * avctx->height){
+    if (s->video_base >= avctx->width * avctx->height) {
         av_log(avctx, AV_LOG_ERROR, "invalid video_base\n");
         return AVERROR_INVALIDDATA;
     }
 
     /** initialize palette */
-    for(i=0;i<AVPALETTE_COUNT;i++)
+    for (i = 0; i < AVPALETTE_COUNT; i++)
         s->palette[i] = 0xFFU << 24 | AV_RB24(&avctx->extradata[6 + i * 3]);
 
     /** decode background frame if present */
     back_size = avctx->extradata_size - EXTRADATA1_SIZE;
 
-    if(back_size > 0){
+    if (back_size > 0) {
         uint8_t *back_frame = av_mallocz(avctx->width*avctx->height);
-        if(!back_frame)
+        if (!back_frame)
             return AVERROR(ENOMEM);
-        rl2_rle_decode(s,avctx->extradata + EXTRADATA1_SIZE,back_size,
-                           back_frame,avctx->width,0);
+        rl2_rle_decode(s, avctx->extradata + EXTRADATA1_SIZE, back_size,
+                       back_frame, avctx->width, 0);
         s->back_frame = back_frame;
     }
     return 0;
@@ -174,25 +175,26 @@ static av_cold int rl2_decode_init(AVCodecContext *avctx)
 
 
 static int rl2_decode_frame(AVCodecContext *avctx,
-                              void *data, int *got_frame,
-                              AVPacket *avpkt)
+                            void *data, int *got_frame,
+                            AVPacket *avpkt)
 {
     const uint8_t *buf = avpkt->data;
-    int ret, buf_size = avpkt->size;
+    int ret, buf_size  = avpkt->size;
     Rl2Context *s = avctx->priv_data;
 
-    if(s->frame.data[0])
+    if (s->frame.data[0])
         avctx->release_buffer(avctx, &s->frame);
 
     /** get buffer */
-    s->frame.reference= 0;
+    s->frame.reference = 0;
     if ((ret = ff_get_buffer(avctx, &s->frame)) < 0) {
         av_log(s->avctx, AV_LOG_ERROR, "get_buffer() failed\n");
         return ret;
     }
 
     /** run length decode */
-    rl2_rle_decode(s,buf,buf_size,s->frame.data[0],s->frame.linesize[0],s->video_base);
+    rl2_rle_decode(s, buf, buf_size, s->frame.data[0], s->frame.linesize[0],
+                   s->video_base);
 
     /** make the palette available on the way out */
     memcpy(s->frame.data[1], s->palette, AVPALETTE_SIZE);
@@ -214,7 +216,7 @@ static av_cold int rl2_decode_end(AVCodecContext *avctx)
 {
     Rl2Context *s = avctx->priv_data;
 
-    if(s->frame.data[0])
+    if (s->frame.data[0])
         avctx->release_buffer(avctx, &s->frame);
 
     av_free(s->back_frame);