diff options
| author | Carl Eugen Hoyos <cehoyos@ag.or.at> | 2013-07-28 19:07:06 +0200 |
|---|---|---|
| committer | Carl Eugen Hoyos <cehoyos@ag.or.at> | 2013-07-28 19:07:06 +0200 |
| commit | 7651c0e49b1f6e66120ef32570a87e3f09224574 (patch) | |
| tree | 69ca29c14ae117bdd2a7e9841c3b009091eece4e /libavcodec | |
| parent | 7e8e8ba9cc2b074b988e8ab28f69cfc5d454d674 (diff) | |
| download | ffmpeg-7651c0e49b1f6e66120ef32570a87e3f09224574.tar.gz | |
Avoid overflows when reading pgm files with maxval != 255 and != 65535.
libjpeg v6b apparently does not initialize the most significant bits
to zero when writing 12bit pgm's with maxval 4095.
Diffstat (limited to 'libavcodec')
| -rw-r--r-- | libavcodec/pnmdec.c | 13 |
1 files changed, 9 insertions, 4 deletions
diff --git a/libavcodec/pnmdec.c b/libavcodec/pnmdec.c index d0c72954aa..96ed83e55d 100644 --- a/libavcodec/pnmdec.c +++ b/libavcodec/pnmdec.c @@ -35,6 +35,7 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data, int i, j, n, linesize, h, upgrade = 0, is_mono = 0; unsigned char *ptr; int components, sample_len, ret; + unsigned int maskval = 0; s->bytestream_start = s->bytestream = (uint8_t *)buf; @@ -75,8 +76,10 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data, n = avctx->width; components=1; sample_len=8; - if (s->maxval < 255) + if (s->maxval < 255) { upgrade = 1; + maskval = (2 << av_log2(s->maxval)) - 1; + } goto do_read; case AV_PIX_FMT_GRAY8A: n = avctx->width * 2; @@ -88,8 +91,10 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data, n = avctx->width * 2; components=1; sample_len=16; - if (s->maxval < 65535) + if (s->maxval < 65535) { upgrade = 2; + maskval = (2 << av_log2(s->maxval)) - 1; + } goto do_read; case AV_PIX_FMT_MONOWHITE: case AV_PIX_FMT_MONOBLACK: @@ -136,11 +141,11 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data, else if (upgrade == 1) { unsigned int j, f = (255 * 128 + s->maxval / 2) / s->maxval; for (j = 0; j < n; j++) - ptr[j] = (s->bytestream[j] * f + 64) >> 7; + ptr[j] = ((s->bytestream[j] & maskval) * f + 64) >> 7; } else if (upgrade == 2) { unsigned int j, v, f = (65535 * 32768 + s->maxval / 2) / s->maxval; for (j = 0; j < n / 2; j++) { - v = av_be2ne16(((uint16_t *)s->bytestream)[j]); + v = av_be2ne16(((uint16_t *)s->bytestream)[j]) & maskval; ((uint16_t *)ptr)[j] = (v * f + 16384) >> 15; } } |