diff options
| author | Josh Allmann <joshua.allmann@gmail.com> | 2024-06-20 17:33:55 -0700 |
|---|---|---|
| committer | Timo Rothenpieler <timo@rothenpieler.org> | 2024-07-01 19:59:01 +0200 |
| commit | c9151ea50715c4ce47ad1c8df519781565db01f6 (patch) | |
| tree | 6e28e11690ad3aa7bbd88a819b0a259e4a32cc6b | |
| parent | cc6fb1643d7e14c6f76a48e0cffad96394cb197c (diff) | |
| download | ffmpeg-c9151ea50715c4ce47ad1c8df519781565db01f6.tar.gz | |
avcodec/nvenc: fix segfault in intra-only mode
In intra-only mode, frameIntervalP is 0, which means the frame
data array is smaller than the number of surfaces.
Together with using the wrong size on deallocation of the
frame_data_array, this lead to a crash.
Signed-off-by: Timo Rothenpieler <timo@rothenpieler.org>
| -rw-r--r-- | libavcodec/nvenc.c | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c index a9945355ba..ab92395ed6 100644 --- a/libavcodec/nvenc.c +++ b/libavcodec/nvenc.c @@ -1021,7 +1021,7 @@ static av_cold int nvenc_recalc_surfaces(AVCodecContext *avctx) // Output in the worst case will only start when the surface buffer is completely full. // Hence we need to keep at least the max amount of surfaces plus the max reorder delay around. - ctx->frame_data_array_nb = ctx->nb_surfaces + ctx->encode_config.frameIntervalP - 1; + ctx->frame_data_array_nb = FFMAX(ctx->nb_surfaces, ctx->nb_surfaces + ctx->encode_config.frameIntervalP - 1); return 0; } @@ -1984,7 +1984,7 @@ av_cold int ff_nvenc_encode_close(AVCodecContext *avctx) av_fifo_freep2(&ctx->unused_surface_queue); if (ctx->frame_data_array) { - for (i = 0; i < ctx->nb_surfaces; i++) + for (i = 0; i < ctx->frame_data_array_nb; i++) av_buffer_unref(&ctx->frame_data_array[i].frame_opaque_ref); av_freep(&ctx->frame_data_array); } |