aboutsummaryrefslogtreecommitdiffstats
path: root/libavcodec/msrledec.c
diff options
context:
space:
mode:
authorMichael Niedermayer <michaelni@gmx.at>2013-04-07 15:42:45 +0200
committerMichael Niedermayer <michaelni@gmx.at>2013-04-07 15:42:45 +0200
commite9d5a6f1c5b65319400a45446ae6523a2296de73 (patch)
tree11110d207fd762b9c52233cde9aba0b32090d0b1 /libavcodec/msrledec.c
parent0af8ed29e4a756c3e207ac65f479c23ca8ed025b (diff)
parent327ff82bac3081d918dceb4931c77e25d0a1480d (diff)
downloadffmpeg-e9d5a6f1c5b65319400a45446ae6523a2296de73.tar.gz
Merge commit '327ff82bac3081d918dceb4931c77e25d0a1480d' into release/0.10
* commit '327ff82bac3081d918dceb4931c77e25d0a1480d': msrle: convert MS RLE decoding function to bytestream2. Update Changelog for the 0.8.6 Release wmaprodec: require block_align to be set. ivi_common: do not call MC for intra frames when dc_transform is unset roqvideodec: fix a potential infinite loop in roqvideo_decode_frame(). Revert "libmp3lame: use the correct remaining buffer size when flushing" lzo: fix overflow checking in copy_backptr() flacdec: simplify bounds checking in flac_probe() atrac3: avoid oversized shifting in decode_bytes() avconv: skip attached files when selecting streams to read from. lavf: fix arithmetic overflows in avformat_seek_file() Conflicts: Changelog avconv.c libavcodec/libmp3lame.c libavcodec/msrledec.c libavformat/utils.c Merged-by: Michael Niedermayer <michaelni@gmx.at>
Diffstat (limited to 'libavcodec/msrledec.c')
-rw-r--r--libavcodec/msrledec.c134
1 files changed, 67 insertions, 67 deletions
diff --git a/libavcodec/msrledec.c b/libavcodec/msrledec.c
index 129f0e0bc0..10bc482329 100644
--- a/libavcodec/msrledec.c
+++ b/libavcodec/msrledec.c
@@ -30,18 +30,9 @@
#include "avcodec.h"
#include "msrledec.h"
-#define FETCH_NEXT_STREAM_BYTE() \
- if (stream_ptr >= data_size) \
- { \
- av_log(avctx, AV_LOG_ERROR, " MS RLE: stream ptr just went out of bounds (1)\n"); \
- return -1; \
- } \
- stream_byte = data[stream_ptr++];
-
static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic,
- const uint8_t *data, int data_size)
+ GetByteContext *gb)
{
- int stream_ptr = 0;
unsigned char rle_code;
unsigned char extra_byte, odd_pixel;
unsigned char stream_byte;
@@ -52,11 +43,16 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic,
int i;
while (row_ptr >= 0) {
- FETCH_NEXT_STREAM_BYTE();
- rle_code = stream_byte;
+ if (bytestream2_get_bytes_left(gb) <= 0) {
+ av_log(avctx, AV_LOG_ERROR,
+ "MS RLE: bytestream overrun, %d rows left\n",
+ row_ptr);
+ return AVERROR_INVALIDDATA;
+ }
+ rle_code = stream_byte = bytestream2_get_byteu(gb);
if (rle_code == 0) {
/* fetch the next byte to see how to handle escape code */
- FETCH_NEXT_STREAM_BYTE();
+ stream_byte = bytestream2_get_byte(gb);
if (stream_byte == 0) {
/* line is done, goto the next one */
row_ptr -= row_dec;
@@ -66,24 +62,26 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic,
return 0;
} else if (stream_byte == 2) {
/* reposition frame decode coordinates */
- FETCH_NEXT_STREAM_BYTE();
+ stream_byte = bytestream2_get_byte(gb);
pixel_ptr += stream_byte;
- FETCH_NEXT_STREAM_BYTE();
+ stream_byte = bytestream2_get_byte(gb);
row_ptr -= stream_byte * row_dec;
} else {
// copy pixels from encoded stream
odd_pixel = stream_byte & 1;
rle_code = (stream_byte + 1) / 2;
extra_byte = rle_code & 0x01;
- if (row_ptr + pixel_ptr + stream_byte > frame_size) {
- av_log(avctx, AV_LOG_ERROR, " MS RLE: frame ptr just went out of bounds (1)\n");
- return -1;
+ if (row_ptr + pixel_ptr + stream_byte > frame_size ||
+ bytestream2_get_bytes_left(gb) < rle_code) {
+ av_log(avctx, AV_LOG_ERROR,
+ "MS RLE: frame/stream ptr just went out of bounds (copy)\n");
+ return AVERROR_INVALIDDATA;
}
for (i = 0; i < rle_code; i++) {
if (pixel_ptr >= avctx->width)
break;
- FETCH_NEXT_STREAM_BYTE();
+ stream_byte = bytestream2_get_byteu(gb);
pic->data[0][row_ptr + pixel_ptr] = stream_byte >> 4;
pixel_ptr++;
if (i + 1 == rle_code && odd_pixel)
@@ -96,15 +94,16 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic,
// if the RLE code is odd, skip a byte in the stream
if (extra_byte)
- stream_ptr++;
+ bytestream2_skip(gb, 1);
}
} else {
// decode a run of data
if (row_ptr + pixel_ptr + stream_byte > frame_size) {
- av_log(avctx, AV_LOG_ERROR, " MS RLE: frame ptr just went out of bounds (1)\n");
- return -1;
+ av_log(avctx, AV_LOG_ERROR,
+ "MS RLE: frame ptr just went out of bounds (run)\n");
+ return AVERROR_INVALIDDATA;
}
- FETCH_NEXT_STREAM_BYTE();
+ stream_byte = bytestream2_get_byte(gb);
for (i = 0; i < rle_code; i++) {
if (pixel_ptr >= avctx->width)
break;
@@ -118,21 +117,21 @@ static int msrle_decode_pal4(AVCodecContext *avctx, AVPicture *pic,
}
/* one last sanity check on the way out */
- if (stream_ptr < data_size) {
- av_log(avctx, AV_LOG_ERROR, " MS RLE: ended frame decode with bytes left over (%d < %d)\n",
- stream_ptr, data_size);
- return -1;
+ if (bytestream2_get_bytes_left(gb)) {
+ av_log(avctx, AV_LOG_ERROR,
+ "MS RLE: ended frame decode with %d bytes left over\n",
+ bytestream2_get_bytes_left(gb));
+ return AVERROR_INVALIDDATA;
}
return 0;
}
-static int msrle_decode_8_16_24_32(AVCodecContext *avctx, AVPicture *pic, int depth,
- const uint8_t *data, int srcsize)
+static int msrle_decode_8_16_24_32(AVCodecContext *avctx, AVPicture *pic,
+ int depth, GetByteContext *gb)
{
uint8_t *output, *output_end;
- const uint8_t* src = data;
int p1, p2, line=avctx->height - 1, pos=0, i;
uint16_t av_uninit(pix16);
uint32_t av_uninit(pix32);
@@ -140,23 +139,30 @@ static int msrle_decode_8_16_24_32(AVCodecContext *avctx, AVPicture *pic, int de
output = pic->data[0] + (avctx->height - 1) * pic->linesize[0];
output_end = pic->data[0] + avctx->height * pic->linesize[0];
- while(src + 1 < data + srcsize) {
- p1 = *src++;
+
+ while (bytestream2_get_bytes_left(gb) > 0) {
+ p1 = bytestream2_get_byteu(gb);
if(p1 == 0) { //Escape code
- p2 = *src++;
+ p2 = bytestream2_get_byte(gb);
if(p2 == 0) { //End-of-line
output = pic->data[0] + (--line) * pic->linesize[0];
- if (line < 0 && !(src+1 < data + srcsize && AV_RB16(src) == 1)) {
- av_log(avctx, AV_LOG_ERROR, "Next line is beyond picture bounds\n");
- return -1;
+ if (line < 0) {
+ if (bytestream2_get_be16(gb) == 1) { // end-of-picture
+ return 0;
+ } else {
+ av_log(avctx, AV_LOG_ERROR,
+ "Next line is beyond picture bounds (%d bytes left)\n",
+ bytestream2_get_bytes_left(gb));
+ return AVERROR_INVALIDDATA;
+ }
}
pos = 0;
continue;
} else if(p2 == 1) { //End-of-picture
return 0;
} else if(p2 == 2) { //Skip
- p1 = *src++;
- p2 = *src++;
+ p1 = bytestream2_get_byte(gb);
+ p2 = bytestream2_get_byte(gb);
line -= p2;
pos += p1;
if (line < 0 || pos >= width){
@@ -167,35 +173,31 @@ static int msrle_decode_8_16_24_32(AVCodecContext *avctx, AVPicture *pic, int de
continue;
}
// Copy data
- if ((pic->linesize[0] > 0 && output + p2 * (depth >> 3) > output_end)
- ||(pic->linesize[0] < 0 && output + p2 * (depth >> 3) < output_end)) {
- src += p2 * (depth >> 3);
+ if ((pic->linesize[0] > 0 && output + p2 * (depth >> 3) > output_end) ||
+ (pic->linesize[0] < 0 && output + p2 * (depth >> 3) < output_end)) {
+ bytestream2_skip(gb, 2 * (depth >> 3));
continue;
+ } else if (bytestream2_get_bytes_left(gb) < p2 * (depth >> 3)) {
+ av_log(avctx, AV_LOG_ERROR, "bytestream overrun\n");
+ return AVERROR_INVALIDDATA;
}
- if(data + srcsize - src < p2 * (depth >> 3)){
- av_log(avctx, AV_LOG_ERROR, "Copy beyond input buffer\n");
- return -1;
- }
+
if ((depth == 8) || (depth == 24)) {
for(i = 0; i < p2 * (depth >> 3); i++) {
- *output++ = *src++;
+ *output++ = bytestream2_get_byteu(gb);
}
// RLE8 copy is actually padded - and runs are not!
if(depth == 8 && (p2 & 1)) {
- src++;
+ bytestream2_skip(gb, 1);
}
} else if (depth == 16) {
for(i = 0; i < p2; i++) {
- pix16 = AV_RL16(src);
- src += 2;
- *(uint16_t*)output = pix16;
+ *(uint16_t*)output = bytestream2_get_le16u(gb);
output += 2;
}
} else if (depth == 32) {
for(i = 0; i < p2; i++) {
- pix32 = AV_RL32(src);
- src += 4;
- *(uint32_t*)output = pix32;
+ *(uint32_t*)output = bytestream2_get_le32u(gb);
output += 4;
}
}
@@ -203,21 +205,19 @@ static int msrle_decode_8_16_24_32(AVCodecContext *avctx, AVPicture *pic, int de
} else { //run of pixels
uint8_t pix[3]; //original pixel
switch(depth){
- case 8: pix[0] = *src++;
+ case 8: pix[0] = bytestream2_get_byte(gb);
break;
- case 16: pix16 = AV_RL16(src);
- src += 2;
+ case 16: pix16 = bytestream2_get_le16(gb);
break;
- case 24: pix[0] = *src++;
- pix[1] = *src++;
- pix[2] = *src++;
+ case 24: pix[0] = bytestream2_get_byte(gb);
+ pix[1] = bytestream2_get_byte(gb);
+ pix[2] = bytestream2_get_byte(gb);
break;
- case 32: pix32 = AV_RL32(src);
- src += 4;
+ case 32: pix32 = bytestream2_get_le32(gb);
break;
}
- if ((pic->linesize[0] > 0 && output + p1 * (depth >> 3) > output_end)
- ||(pic->linesize[0] < 0 && output + p1 * (depth >> 3) < output_end))
+ if ((pic->linesize[0] > 0 && output + p1 * (depth >> 3) > output_end) ||
+ (pic->linesize[0] < 0 && output + p1 * (depth >> 3) < output_end))
continue;
for(i = 0; i < p1; i++) {
switch(depth){
@@ -244,17 +244,17 @@ static int msrle_decode_8_16_24_32(AVCodecContext *avctx, AVPicture *pic, int de
}
-int ff_msrle_decode(AVCodecContext *avctx, AVPicture *pic, int depth,
- const uint8_t* data, int data_size)
+int ff_msrle_decode(AVCodecContext *avctx, AVPicture *pic,
+ int depth, GetByteContext *gb)
{
switch(depth){
case 4:
- return msrle_decode_pal4(avctx, pic, data, data_size);
+ return msrle_decode_pal4(avctx, pic, gb);
case 8:
case 16:
case 24:
case 32:
- return msrle_decode_8_16_24_32(avctx, pic, depth, data, data_size);
+ return msrle_decode_8_16_24_32(avctx, pic, depth, gb);
default:
av_log(avctx, AV_LOG_ERROR, "Unknown depth %d\n", depth);
return -1;