aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAndreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>2016-10-13 22:14:46 +0200
committerAndreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>2016-10-17 17:24:40 +0200
commit0887f1835d69d65fef45eb9aa0183bb06954f52a (patch)
treeb9742975ee8a664756fca934a3b9970d5a2bc495
parentde42af2beef7ecc3bf44d746c316de437b8be274 (diff)
downloadffmpeg-0887f1835d69d65fef45eb9aa0183bb06954f52a.tar.gz
libopenjpegenc: fix out-of-bounds reads when filling the edges
The calculation of width/height should round up, not round down to prevent setting width or height to 0. Also image->comps[compno].w is unsigned (at least in openjpeg2), so the calculation could silently wrap around without the explicit cast to int. Reviewed-by: Michael Bradshaw <mjbshaw@gmail.com> Reviewed-by: Michael Niedermayer <michael@niedermayer.cc> Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun@googlemail.com> (cherry picked from commit 56706ac0d5723cb549fec2602e798ab1bf6004cd) Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>
-rw-r--r--libavcodec/libopenjpegenc.c18
1 files changed, 9 insertions, 9 deletions
diff --git a/libavcodec/libopenjpegenc.c b/libavcodec/libopenjpegenc.c
index a5c3e961f7..2ca7b2d4d2 100644
--- a/libavcodec/libopenjpegenc.c
+++ b/libavcodec/libopenjpegenc.c
@@ -421,7 +421,7 @@ static int libopenjpeg_copy_packed8(AVCodecContext *avctx, const AVFrame *frame,
for (; y < image->comps[compno].h; ++y) {
image_line = image->comps[compno].data + y * image->comps[compno].w;
for (x = 0; x < image->comps[compno].w; ++x) {
- image_line[x] = image_line[x - image->comps[compno].w];
+ image_line[x] = image_line[x - (int)image->comps[compno].w];
}
}
}
@@ -461,7 +461,7 @@ static int libopenjpeg_copy_packed12(AVCodecContext *avctx, const AVFrame *frame
for (; y < image->comps[compno].h; ++y) {
image_line = image->comps[compno].data + y * image->comps[compno].w;
for (x = 0; x < image->comps[compno].w; ++x) {
- image_line[x] = image_line[x - image->comps[compno].w];
+ image_line[x] = image_line[x - (int)image->comps[compno].w];
}
}
}
@@ -501,7 +501,7 @@ static int libopenjpeg_copy_packed16(AVCodecContext *avctx, const AVFrame *frame
for (; y < image->comps[compno].h; ++y) {
image_line = image->comps[compno].data + y * image->comps[compno].w;
for (x = 0; x < image->comps[compno].w; ++x) {
- image_line[x] = image_line[x - image->comps[compno].w];
+ image_line[x] = image_line[x - (int)image->comps[compno].w];
}
}
}
@@ -528,8 +528,8 @@ static int libopenjpeg_copy_unpacked8(AVCodecContext *avctx, const AVFrame *fram
}
for (compno = 0; compno < numcomps; ++compno) {
- width = avctx->width / image->comps[compno].dx;
- height = avctx->height / image->comps[compno].dy;
+ width = (avctx->width + image->comps[compno].dx - 1) / image->comps[compno].dx;
+ height = (avctx->height + image->comps[compno].dy - 1) / image->comps[compno].dy;
for (y = 0; y < height; ++y) {
image_line = image->comps[compno].data + y * image->comps[compno].w;
frame_index = y * frame->linesize[compno];
@@ -542,7 +542,7 @@ static int libopenjpeg_copy_unpacked8(AVCodecContext *avctx, const AVFrame *fram
for (; y < image->comps[compno].h; ++y) {
image_line = image->comps[compno].data + y * image->comps[compno].w;
for (x = 0; x < image->comps[compno].w; ++x) {
- image_line[x] = image_line[x - image->comps[compno].w];
+ image_line[x] = image_line[x - (int)image->comps[compno].w];
}
}
}
@@ -570,8 +570,8 @@ static int libopenjpeg_copy_unpacked16(AVCodecContext *avctx, const AVFrame *fra
}
for (compno = 0; compno < numcomps; ++compno) {
- width = avctx->width / image->comps[compno].dx;
- height = avctx->height / image->comps[compno].dy;
+ width = (avctx->width + image->comps[compno].dx - 1) / image->comps[compno].dx;
+ height = (avctx->height + image->comps[compno].dy - 1) / image->comps[compno].dy;
frame_ptr = (uint16_t *)frame->data[compno];
for (y = 0; y < height; ++y) {
image_line = image->comps[compno].data + y * image->comps[compno].w;
@@ -585,7 +585,7 @@ static int libopenjpeg_copy_unpacked16(AVCodecContext *avctx, const AVFrame *fra
for (; y < image->comps[compno].h; ++y) {
image_line = image->comps[compno].data + y * image->comps[compno].w;
for (x = 0; x < image->comps[compno].w; ++x) {
- image_line[x] = image_line[x - image->comps[compno].w];
+ image_line[x] = image_line[x - (int)image->comps[compno].w];
}
}
}