diff options
| author | Paul B Mahol <onemda@gmail.com> | 2022-03-05 11:10:24 +0100 |
|---|---|---|
| committer | Paul B Mahol <onemda@gmail.com> | 2022-03-05 22:11:38 +0100 |
| commit | 044c09c0a0b5e2d91d75619c1327e232a4914030 (patch) | |
| tree | 33bc3404adaa44adcf4661205baed9696de929f4 | |
| parent | 37480b1b85b0405563962b581dc2899b1b4bec59 (diff) | |
| download | ffmpeg-044c09c0a0b5e2d91d75619c1327e232a4914030.tar.gz | |
avcodec/dnxhdenc: retry increasing qscale to not overflow max_bits
Increase mb_bits type from uint16_t to uint32_t to fix possible overflows
in bit size calculations.
Update fate test that needs change.
| -rw-r--r-- | libavcodec/dnxhdenc.c | 8 | ||||
| -rw-r--r-- | libavcodec/dnxhdenc.h | 2 | ||||
| -rw-r--r-- | tests/ref/lavf/mxf_opatom | 4 |
3 files changed, 8 insertions, 6 deletions
diff --git a/libavcodec/dnxhdenc.c b/libavcodec/dnxhdenc.c index 374fec499b..9bdecec284 100644 --- a/libavcodec/dnxhdenc.c +++ b/libavcodec/dnxhdenc.c @@ -924,7 +924,7 @@ static void dnxhd_setup_threads_slices(DNXHDEncContext *ctx) unsigned mb = mb_y * ctx->m.mb_width + mb_x; ctx->slice_size[mb_y] += ctx->mb_bits[mb]; } - ctx->slice_size[mb_y] = (ctx->slice_size[mb_y] + 31) & ~31; + ctx->slice_size[mb_y] = (ctx->slice_size[mb_y] + 31U) & ~31U; ctx->slice_size[mb_y] >>= 3; thread_size = ctx->slice_size[mb_y]; offset += thread_size; @@ -1220,17 +1220,19 @@ static int dnxhd_encode_fast(AVCodecContext *avctx, DNXHDEncContext *ctx) avctx->execute2(avctx, dnxhd_mb_var_thread, NULL, NULL, ctx->m.mb_height); radix_sort(ctx->mb_cmp, ctx->mb_cmp_tmp, ctx->m.mb_num); +retry: for (x = 0; x < ctx->m.mb_num && max_bits > ctx->frame_bits; x++) { int mb = ctx->mb_cmp[x].mb; int rc = (ctx->qscale * ctx->m.mb_num ) + mb; max_bits -= ctx->mb_rc[rc].bits - ctx->mb_rc[rc + ctx->m.mb_num].bits; - ctx->mb_qscale[mb] = ctx->qscale + 1; + if (ctx->mb_qscale[mb] < 255) + ctx->mb_qscale[mb]++; ctx->mb_bits[mb] = ctx->mb_rc[rc + ctx->m.mb_num].bits; } if (max_bits > ctx->frame_bits) - return AVERROR(EINVAL); + goto retry; } return 0; } diff --git a/libavcodec/dnxhdenc.h b/libavcodec/dnxhdenc.h index 30ae8c15e3..7726a3915f 100644 --- a/libavcodec/dnxhdenc.h +++ b/libavcodec/dnxhdenc.h @@ -100,7 +100,7 @@ typedef struct DNXHDEncContext { unsigned qscale; unsigned lambda; - uint16_t *mb_bits; + uint32_t *mb_bits; uint8_t *mb_qscale; RCCMPEntry *mb_cmp; diff --git a/tests/ref/lavf/mxf_opatom b/tests/ref/lavf/mxf_opatom index e34cf2559e..359422ce9a 100644 --- a/tests/ref/lavf/mxf_opatom +++ b/tests/ref/lavf/mxf_opatom @@ -1,3 +1,3 @@ -aab6397829bd90f0c77a3f9fde53bb9c *tests/data/lavf/lavf.mxf_opatom +b1e32792b835ac51b0b1d5dcb9497e4e *tests/data/lavf/lavf.mxf_opatom 4717625 tests/data/lavf/lavf.mxf_opatom -tests/data/lavf/lavf.mxf_opatom CRC=0xf55aa22a +tests/data/lavf/lavf.mxf_opatom CRC=0xb13ba2f8 |