avcodec/cabac: Check initial cabac decoder state

Fixes integer overflows
Fixes: 1430e9c43fae47a24c179c7c54f94918/signal_sigsegv_421427_2340_591e9810c7b09efe501ad84638c9e9f8.264

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Found-by: xiedingbao (Ticket4727)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 8000d484b83aafa752d84fbdbfb352ffe0dc64f8)

Conflicts:

	libavcodec/cabac.h

5 files changed, 14 insertions, 5 deletions

diff --git a/libavcodec/cabac.c b/libavcodec/cabac.c
index 8cc9333e09..f298336ea4 100644
--- a/libavcodec/cabac.c
+++ b/libavcodec/cabac.c
@@ -51,7 +51,7 @@ void ff_init_cabac_encoder(CABACContext *c, uint8_t *buf, int buf_size){
  *
  * @param buf_size size of buf in bits
  */
-void ff_init_cabac_decoder(CABACContext *c, const uint8_t *buf, int buf_size){
+int ff_init_cabac_decoder(CABACContext *c, const uint8_t *buf, int buf_size){
     c->bytestream_start=
     c->bytestream= buf;
     c->bytestream_end= buf + buf_size;
@@ -64,6 +64,9 @@ void ff_init_cabac_decoder(CABACContext *c, const uint8_t *buf, int buf_size){
 #endif
     c->low+= ((*c->bytestream++)<<2) + 2;
     c->range= 0x1FE;
+    if ((c->range<<(CABAC_BITS+1)) < c->low)
+        return AVERROR_INVALIDDATA;
+    return 0;
 }
 
 void ff_init_cabac_states(void)
diff --git a/libavcodec/cabac.h b/libavcodec/cabac.h
index f9eafed105..857211c9d9 100644
--- a/libavcodec/cabac.h
+++ b/libavcodec/cabac.h
@@ -56,7 +56,7 @@ typedef struct CABACContext{
 }CABACContext;
 
 void ff_init_cabac_encoder(CABACContext *c, uint8_t *buf, int buf_size);
-void ff_init_cabac_decoder(CABACContext *c, const uint8_t *buf, int buf_size);
+int ff_init_cabac_decoder(CABACContext *c, const uint8_t *buf, int buf_size);
 void ff_init_cabac_states(void);
 
 #endif /* AVCODEC_CABAC_H */
diff --git a/libavcodec/cabac_functions.h b/libavcodec/cabac_functions.h
index 4e132535e1..2d1d2a6b89 100644
--- a/libavcodec/cabac_functions.h
+++ b/libavcodec/cabac_functions.h
@@ -191,7 +191,8 @@ static av_unused const uint8_t* skip_bytes(CABACContext *c, int n) {
 #endif
     if ((int) (c->bytestream_end - ptr) < n)
         return NULL;
-    ff_init_cabac_decoder(c, ptr + n, c->bytestream_end - ptr - n);
+    if (ff_init_cabac_decoder(c, ptr + n, c->bytestream_end - ptr - n) < 0)
+        return NULL;
 
     return ptr;
 }
diff --git a/libavcodec/h264_cabac.c b/libavcodec/h264_cabac.c
index c1c8b80855..04d412b74b 100644
--- a/libavcodec/h264_cabac.c
+++ b/libavcodec/h264_cabac.c
@@ -2026,6 +2026,7 @@ decode_intra_mb:
         const int mb_size = ff_h264_mb_sizes[h->sps.chroma_format_idc] *
                             h->sps.bit_depth_luma >> 3;
         const uint8_t *ptr;
+        int ret;
 
         // We assume these blocks are very rare so we do not optimize it.
         // FIXME The two following lines get the bitstream position in the cabac
@@ -2042,7 +2043,9 @@ decode_intra_mb:
         sl->intra_pcm_ptr = ptr;
         ptr += mb_size;
 
-        ff_init_cabac_decoder(&sl->cabac, ptr, sl->cabac.bytestream_end - ptr);
+        ret = ff_init_cabac_decoder(&sl->cabac, ptr, sl->cabac.bytestream_end - ptr);
+        if (ret < 0)
+            return ret;
 
         // All blocks are present
         h->cbp_table[mb_xy] = 0xf7ef;
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index 041acfcd35..843cfd0979 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -2372,9 +2372,11 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg)
         align_get_bits(&sl->gb);
 
         /* init cabac */
-        ff_init_cabac_decoder(&sl->cabac,
+        ret = ff_init_cabac_decoder(&sl->cabac,
                               sl->gb.buffer + get_bits_count(&sl->gb) / 8,
                               (get_bits_left(&sl->gb) + 7) / 8);
+        if (ret < 0)
+            return ret;
 
         ff_h264_init_cabac_states(h, sl);