avcodec/dnxhdenc: retry increasing qscale to not overflow max_bits

Increase mb_bits type from uint16_t to uint32_t to fix possible overflows
in bit size calculations.

Update fate test that needs change.

2 files changed, 6 insertions, 4 deletions

diff --git a/libavcodec/dnxhdenc.c b/libavcodec/dnxhdenc.c
index 374fec499b..9bdecec284 100644
--- a/libavcodec/dnxhdenc.c
+++ b/libavcodec/dnxhdenc.c
@@ -924,7 +924,7 @@ static void dnxhd_setup_threads_slices(DNXHDEncContext *ctx)
             unsigned mb = mb_y * ctx->m.mb_width + mb_x;
             ctx->slice_size[mb_y] += ctx->mb_bits[mb];
         }
-        ctx->slice_size[mb_y]   = (ctx->slice_size[mb_y] + 31) & ~31;
+        ctx->slice_size[mb_y]   = (ctx->slice_size[mb_y] + 31U) & ~31U;
         ctx->slice_size[mb_y] >>= 3;
         thread_size = ctx->slice_size[mb_y];
         offset += thread_size;
@@ -1220,17 +1220,19 @@ static int dnxhd_encode_fast(AVCodecContext *avctx, DNXHDEncContext *ctx)
             avctx->execute2(avctx, dnxhd_mb_var_thread,
                             NULL, NULL, ctx->m.mb_height);
         radix_sort(ctx->mb_cmp, ctx->mb_cmp_tmp, ctx->m.mb_num);
+retry:
         for (x = 0; x < ctx->m.mb_num && max_bits > ctx->frame_bits; x++) {
             int mb = ctx->mb_cmp[x].mb;
             int rc = (ctx->qscale * ctx->m.mb_num ) + mb;
             max_bits -= ctx->mb_rc[rc].bits -
                         ctx->mb_rc[rc + ctx->m.mb_num].bits;
-            ctx->mb_qscale[mb] = ctx->qscale + 1;
+            if (ctx->mb_qscale[mb] < 255)
+                ctx->mb_qscale[mb]++;
             ctx->mb_bits[mb]   = ctx->mb_rc[rc + ctx->m.mb_num].bits;
         }
 
         if (max_bits > ctx->frame_bits)
-            return AVERROR(EINVAL);
+            goto retry;
     }
     return 0;
 }
diff --git a/libavcodec/dnxhdenc.h b/libavcodec/dnxhdenc.h
index 30ae8c15e3..7726a3915f 100644
--- a/libavcodec/dnxhdenc.h
+++ b/libavcodec/dnxhdenc.h
@@ -100,7 +100,7 @@ typedef struct DNXHDEncContext {
     unsigned qscale;
     unsigned lambda;
 
-    uint16_t *mb_bits;
+    uint32_t *mb_bits;
     uint8_t  *mb_qscale;
 
     RCCMPEntry *mb_cmp;