checking bitstream values and other related changes

some of these might have been exploitable

Originally committed as revision 7591 to svn://svn.ffmpeg.org/ffmpeg/trunk

1 files changed, 118 insertions, 50 deletions

diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 1dc6fafd02..00c3573ea1 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -122,7 +122,7 @@ typedef struct PPS{
     int pic_order_present;      ///< pic_order_present_flag
     int slice_group_count;      ///< num_slice_groups_minus1 + 1
     int mb_slice_group_map_type;
-    int ref_count[2];           ///< num_ref_idx_l0/1_active_minus1 + 1
+    unsigned int ref_count[2];  ///< num_ref_idx_l0/1_active_minus1 + 1
     int weighted_pred;          ///< weighted_pred_flag
     int weighted_bipred_idc;
     int init_qp;                ///< pic_init_qp_minus26 + 26
@@ -274,7 +274,7 @@ typedef struct H264Context{
     int mb_field_decoding_flag;
     int mb_mbaff;              ///< mb_aff_frame && mb_field_decoding_flag
 
-    int sub_mb_type[4];
+    unsigned int sub_mb_type[4];
 
     //POC stuff
     int poc_lsb;
@@ -325,7 +325,7 @@ typedef struct H264Context{
     /**
      * num_ref_idx_l0/1_active_minus1 + 1
      */
-    int ref_count[2];            ///< counts frames or fields, depending on current mb mode
+    unsigned int ref_count[2];   ///< counts frames or fields, depending on current mb mode
     Picture *short_ref[32];
     Picture *long_ref[32];
     Picture default_ref_list[2][32];
@@ -349,6 +349,7 @@ typedef struct H264Context{
     GetBitContext *inter_gb_ptr;
 
     DECLARE_ALIGNED_8(DCTELEM, mb[16*24]);
+    DCTELEM mb_padding[256];        ///< as mb is addressed by scantable[i] and scantable is uint8_t we can either check that i is not to large or ensure that there is some unused stuff after mb
 
     /**
      * Cabac
@@ -1398,7 +1399,7 @@ static inline void pred_direct_motion(H264Context * const h, int *mb_type){
     const int8_t *l1ref0 = &h->ref_list[1][0].ref_index[0][b8_xy];
     const int8_t *l1ref1 = &h->ref_list[1][0].ref_index[1][b8_xy];
     const int is_b8x8 = IS_8X8(*mb_type);
-    int sub_mb_type;
+    unsigned int sub_mb_type;
     int i8, i4;
 
 #define MB_TYPE_16x16_OR_INTRA (MB_TYPE_16x16|MB_TYPE_INTRA4x4|MB_TYPE_INTRA16x16|MB_TYPE_INTRA_PCM)
@@ -3916,8 +3917,8 @@ static int decode_ref_pic_list_reordering(H264Context *h){
             int pred= h->curr_pic_num;
 
             for(index=0; ; index++){
-                int reordering_of_pic_nums_idc= get_ue_golomb(&s->gb);
-                int pic_id;
+                unsigned int reordering_of_pic_nums_idc= get_ue_golomb(&s->gb);
+                unsigned int pic_id;
                 int i;
                 Picture *ref = NULL;
 
@@ -3931,7 +3932,7 @@ static int decode_ref_pic_list_reordering(H264Context *h){
 
                 if(reordering_of_pic_nums_idc<3){
                     if(reordering_of_pic_nums_idc<2){
-                        const int abs_diff_pic_num= get_ue_golomb(&s->gb) + 1;
+                        const unsigned int abs_diff_pic_num= get_ue_golomb(&s->gb) + 1;
 
                         if(abs_diff_pic_num >= h->max_pic_num){
                             av_log(h->s.avctx, AV_LOG_ERROR, "abs_diff_pic_num overflow\n");
@@ -3953,6 +3954,10 @@ static int decode_ref_pic_list_reordering(H264Context *h){
                             ref->pic_id= ref->frame_num;
                     }else{
                         pic_id= get_ue_golomb(&s->gb); //long_term_pic_idx
+                        if(pic_id>31){
+                            av_log(h->s.avctx, AV_LOG_ERROR, "long_term_pic_idx overflow\n");
+                            return -1;
+                        }
                         ref = h->long_ref[pic_id];
                         if(ref){
                             ref->pic_id= pic_id;
@@ -4358,14 +4363,15 @@ static int decode_ref_pic_marking(H264Context *h){
                     }*/
                 }
                 if(opcode==MMCO_SHORT2LONG || opcode==MMCO_LONG2UNUSED || opcode==MMCO_LONG || opcode==MMCO_SET_MAX_LONG){
-                    h->mmco[i].long_index= get_ue_golomb(&s->gb);
-                    if(/*h->mmco[i].long_index >= h->long_ref_count || h->long_ref[ h->mmco[i].long_index ] == NULL*/ h->mmco[i].long_index >= 16){
+                    unsigned int long_index= get_ue_golomb(&s->gb);
+                    if(/*h->mmco[i].long_index >= h->long_ref_count || h->long_ref[ h->mmco[i].long_index ] == NULL*/ long_index >= 16){
                         av_log(h->s.avctx, AV_LOG_ERROR, "illegal long ref in memory management control operation %d\n", opcode);
                         return -1;
                     }
+                    h->mmco[i].long_index= long_index;
                 }
 
-                if(opcode > MMCO_LONG){
+                if(opcode > (unsigned)MMCO_LONG){
                     av_log(h->s.avctx, AV_LOG_ERROR, "illegal memory management control operation %d\n", opcode);
                     return -1;
                 }
@@ -4483,11 +4489,11 @@ static int init_poc(H264Context *h){
  */
 static int decode_slice_header(H264Context *h){
     MpegEncContext * const s = &h->s;
-    int first_mb_in_slice;
+    unsigned int first_mb_in_slice;
     unsigned int pps_id;
     int num_ref_idx_active_override_flag;
     static const uint8_t slice_type_map[5]= {P_TYPE, B_TYPE, I_TYPE, SP_TYPE, SI_TYPE};
-    int slice_type;
+    unsigned int slice_type, tmp;
     int default_ref_list_done = 0;
 
     s->current_picture.reference= h->nal_ref_idc != 0;
@@ -4640,12 +4646,15 @@ static int decode_slice_header(H264Context *h){
             h->mb_aff_frame = h->sps.mb_aff;
         }
     }
-
-    s->resync_mb_x = s->mb_x = first_mb_in_slice % s->mb_width;
-    s->resync_mb_y = s->mb_y = (first_mb_in_slice / s->mb_width) << h->mb_aff_frame;
-    if(s->mb_y >= s->mb_height){
+    assert(s->mb_num == s->mb_width * s->mb_height);
+    if(first_mb_in_slice << h->mb_aff_frame >= s->mb_num ||
+       first_mb_in_slice                    >= s->mb_num){
+        av_log(h->s.avctx, AV_LOG_ERROR, "first_mb_in_slice overflow\n");
         return -1;
     }
+    s->resync_mb_x = s->mb_x = first_mb_in_slice % s->mb_width;
+    s->resync_mb_y = s->mb_y = (first_mb_in_slice / s->mb_width) << h->mb_aff_frame;
+    assert(s->mb_y < s->mb_height);
 
     if(s->picture_structure==PICT_FRAME){
         h->curr_pic_num=   h->frame_num;
@@ -4699,6 +4708,7 @@ static int decode_slice_header(H264Context *h){
 
             if(h->ref_count[0] > 32 || h->ref_count[1] > 32){
                 av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow\n");
+                h->ref_count[0]= h->ref_count[1]= 1;
                 return -1;
             }
         }
@@ -4725,15 +4735,22 @@ static int decode_slice_header(H264Context *h){
     if(FRAME_MBAFF)
         fill_mbaff_ref_list(h);
 
-    if( h->slice_type != I_TYPE && h->slice_type != SI_TYPE && h->pps.cabac )
-        h->cabac_init_idc = get_ue_golomb(&s->gb);
+    if( h->slice_type != I_TYPE && h->slice_type != SI_TYPE && h->pps.cabac ){
+        tmp = get_ue_golomb(&s->gb);
+        if(tmp > 2){
+            av_log(s->avctx, AV_LOG_ERROR, "cabac_init_idc overflow\n");
+            return -1;
+        }
+        h->cabac_init_idc= tmp;
+    }
 
     h->last_qscale_diff = 0;
-    s->qscale = h->pps.init_qp + get_se_golomb(&s->gb);
-    if(s->qscale<0 || s->qscale>51){
-        av_log(s->avctx, AV_LOG_ERROR, "QP %d out of range\n", s->qscale);
+    tmp = h->pps.init_qp + get_se_golomb(&s->gb);
+    if(tmp>51){
+        av_log(s->avctx, AV_LOG_ERROR, "QP %u out of range\n", tmp);
         return -1;
     }
+    s->qscale= tmp;
     h->chroma_qp = get_chroma_qp(h->pps.chroma_qp_index_offset, s->qscale);
     //FIXME qscale / qp ... stuff
     if(h->slice_type == SP_TYPE){
@@ -4747,7 +4764,12 @@ static int decode_slice_header(H264Context *h){
     h->slice_alpha_c0_offset = 0;
     h->slice_beta_offset = 0;
     if( h->pps.deblocking_filter_parameters_present ) {
-        h->deblocking_filter= get_ue_golomb(&s->gb);
+        tmp= get_ue_golomb(&s->gb);
+        if(tmp > 2){
+            av_log(s->avctx, AV_LOG_ERROR, "deblocking_filter_idc %u out of range\n", tmp);
+            return -1;
+        }
+        h->deblocking_filter= tmp;
         if(h->deblocking_filter < 2)
             h->deblocking_filter^= 1; // 1<->0
 
@@ -5044,7 +5066,8 @@ static void decode_mb_skip(H264Context *h){
 static int decode_mb_cavlc(H264Context *h){
     MpegEncContext * const s = &h->s;
     const int mb_xy= s->mb_x + s->mb_y*s->mb_stride;
-    int mb_type, partition_count, cbp;
+    int partition_count;
+    unsigned int mb_type, cbp;
     int dct8x8_allowed= h->pps.transform_8x8_mode;
 
     s->dsp.clear_blocks(h->mb); //FIXME avoid if already clear (move after skip handlong?
@@ -5158,6 +5181,7 @@ decode_intra_mb:
 
     //mb_pred
     if(IS_INTRA(mb_type)){
+            int pred_mode;
 //            init_top_left_availability(h);
             if(IS_INTRA4x4(mb_type)){
                 int i;
@@ -5189,11 +5213,11 @@ decode_intra_mb:
                 if(h->intra16x16_pred_mode < 0)
                     return -1;
             }
-            h->chroma_pred_mode= get_ue_golomb(&s->gb);
 
-            h->chroma_pred_mode= check_intra_pred_mode(h, h->chroma_pred_mode);
-            if(h->chroma_pred_mode < 0)
+            pred_mode= check_intra_pred_mode(h, get_ue_golomb(&s->gb));
+            if(pred_mode < 0)
                 return -1;
+            h->chroma_pred_mode= pred_mode;
     }else if(partition_count==4){
         int i, j, sub_partition_count[4], list, ref[2][4];
 
@@ -5201,7 +5225,7 @@ decode_intra_mb:
             for(i=0; i<4; i++){
                 h->sub_mb_type[i]= get_ue_golomb(&s->gb);
                 if(h->sub_mb_type[i] >=13){
-                    av_log(h->s.avctx, AV_LOG_ERROR, "B sub_mb_type %d out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y);
+                    av_log(h->s.avctx, AV_LOG_ERROR, "B sub_mb_type %u out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y);
                     return -1;
                 }
                 sub_partition_count[i]= b_sub_mb_type_info[ h->sub_mb_type[i] ].partition_count;
@@ -5220,7 +5244,7 @@ decode_intra_mb:
             for(i=0; i<4; i++){
                 h->sub_mb_type[i]= get_ue_golomb(&s->gb);
                 if(h->sub_mb_type[i] >=4){
-                    av_log(h->s.avctx, AV_LOG_ERROR, "P sub_mb_type %d out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y);
+                    av_log(h->s.avctx, AV_LOG_ERROR, "P sub_mb_type %u out of range at %d %d\n", h->sub_mb_type[i], s->mb_x, s->mb_y);
                     return -1;
                 }
                 sub_partition_count[i]= p_sub_mb_type_info[ h->sub_mb_type[i] ].partition_count;
@@ -5234,7 +5258,12 @@ decode_intra_mb:
             for(i=0; i<4; i++){
                 if(IS_DIRECT(h->sub_mb_type[i])) continue;
                 if(IS_DIR(h->sub_mb_type[i], 0, list)){
-                    ref[list][i] = get_te0_golomb(&s->gb, ref_count); //FIXME init to 0 before and skip?
+                    unsigned int tmp = get_te0_golomb(&s->gb, ref_count); //FIXME init to 0 before and skip?
+                    if(tmp>=ref_count){
+                        av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", tmp);
+                        return -1;
+                    }
+                    ref[list][i]= tmp;
                 }else{
                  //FIXME
                     ref[list][i] = -1;
@@ -5303,7 +5332,11 @@ decode_intra_mb:
             for(list=0; list<2; list++){
                 if(h->ref_count[list]>0){
                     if(IS_DIR(mb_type, 0, list)){
-                        const int val= get_te0_golomb(&s->gb, h->ref_count[list]);
+                        unsigned int val= get_te0_golomb(&s->gb, h->ref_count[list]);
+                        if(val >= h->ref_count[list]){
+                            av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", val);
+                            return -1;
+                        }
                         fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, val, 1);
                     }else
                         fill_rectangle(&h->ref_cache[list][ scan8[0] ], 4, 4, 8, (LIST_NOT_USED&0xFF), 1);
@@ -5326,7 +5359,11 @@ decode_intra_mb:
                 if(h->ref_count[list]>0){
                     for(i=0; i<2; i++){
                         if(IS_DIR(mb_type, i, list)){
-                            const int val= get_te0_golomb(&s->gb, h->ref_count[list]);
+                            unsigned int val= get_te0_golomb(&s->gb, h->ref_count[list]);
+                            if(val >= h->ref_count[list]){
+                                av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", val);
+                                return -1;
+                            }
                             fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, val, 1);
                         }else
                             fill_rectangle(&h->ref_cache[list][ scan8[0] + 16*i ], 4, 2, 8, (LIST_NOT_USED&0xFF), 1);
@@ -5352,7 +5389,11 @@ decode_intra_mb:
                 if(h->ref_count[list]>0){
                     for(i=0; i<2; i++){
                         if(IS_DIR(mb_type, i, list)){ //FIXME optimize
-                            const int val= get_te0_golomb(&s->gb, h->ref_count[list]);
+                            unsigned int val= get_te0_golomb(&s->gb, h->ref_count[list]);
+                            if(val >= h->ref_count[list]){
+                                av_log(h->s.avctx, AV_LOG_ERROR, "ref %u overflow\n", val);
+                                return -1;
+                            }
                             fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, val, 1);
                         }else
                             fill_rectangle(&h->ref_cache[list][ scan8[0] + 2*i ], 2, 4, 8, (LIST_NOT_USED&0xFF), 1);
@@ -5381,7 +5422,7 @@ decode_intra_mb:
     if(!IS_INTRA16x16(mb_type)){
         cbp= get_ue_golomb(&s->gb);
         if(cbp > 47){
-            av_log(h->s.avctx, AV_LOG_ERROR, "cbp too large (%d) at %d %d\n", cbp, s->mb_x, s->mb_y);
+            av_log(h->s.avctx, AV_LOG_ERROR, "cbp too large (%u) at %d %d\n", cbp, s->mb_x, s->mb_y);
             return -1;
         }
 
@@ -5863,6 +5904,10 @@ static int decode_cabac_mb_ref( H264Context *h, int list, int n ) {
             ctx = 4;
         else
             ctx = 5;
+        if(ref >= 32 /*h->ref_list[list]*/){
+            av_log(h->s.avctx, AV_LOG_ERROR, "overflow in decode_cabac_mb_ref\n");
+            return 0; //FIXME we should return -1 and check the return everywhere
+        }
     }
     return ref;
 }
@@ -5896,6 +5941,10 @@ static int decode_cabac_mb_mvd( H264Context *h, int list, int n, int l ) {
         while( get_cabac_bypass( &h->cabac ) ) {
             mvd += 1 << k;
             k++;
+            if(k>24){
+                av_log(h->s.avctx, AV_LOG_ERROR, "overflow in decode_cabac_mb_mvd\n");
+                return INT_MIN;
+            }
         }
         while( k-- ) {
             if( get_cabac_bypass( &h->cabac ) )
@@ -6285,7 +6334,7 @@ decode_intra_mb:
     fill_caches(h, mb_type, 0);
 
     if( IS_INTRA( mb_type ) ) {
-        int i;
+        int i, pred_mode;
         if( IS_INTRA4x4( mb_type ) ) {
             if( dct8x8_allowed && decode_cabac_mb_transform_size( h ) ) {
                 mb_type |= MB_TYPE_8x8DCT;
@@ -6309,10 +6358,11 @@ decode_intra_mb:
             if( h->intra16x16_pred_mode < 0 ) return -1;
         }
         h->chroma_pred_mode_table[mb_xy] =
-            h->chroma_pred_mode          = decode_cabac_mb_chroma_pre_mode( h );
+        pred_mode                        = decode_cabac_mb_chroma_pre_mode( h );
 
-        h->chroma_pred_mode= check_intra_pred_mode( h, h->chroma_pred_mode );
-        if( h->chroma_pred_mode < 0 ) return -1;
+        pred_mode= check_intra_pred_mode( h, pred_mode );
+        if( pred_mode < 0 ) return -1;
+        h->chroma_pred_mode= pred_mode;
     } else if( partition_count == 4 ) {
         int i, j, sub_partition_count[4], list, ref[2][4];
 
@@ -7561,7 +7611,8 @@ static inline void decode_hrd_parameters(H264Context *h, SPS *sps){
 
 static inline int decode_vui_parameters(H264Context *h, SPS *sps){
     MpegEncContext * const s = &h->s;
-    int aspect_ratio_info_present_flag, aspect_ratio_idc;
+    int aspect_ratio_info_present_flag;
+    unsigned int aspect_ratio_idc;
     int nal_hrd_parameters_present_flag, vcl_hrd_parameters_present_flag;
 
     aspect_ratio_info_present_flag= get_bits1(&s->gb);
@@ -7683,7 +7734,7 @@ static void decode_scaling_matrices(H264Context *h, SPS *sps, PPS *pps, int is_s
 static inline int decode_seq_parameter_set(H264Context *h){
     MpegEncContext * const s = &h->s;
     int profile_idc, level_idc;
-    unsigned int sps_id;
+    unsigned int sps_id, tmp, mb_width, mb_height;
     int i;
     SPS *sps;
 
@@ -7725,26 +7776,36 @@ static inline int decode_seq_parameter_set(H264Context *h){
         sps->delta_pic_order_always_zero_flag= get_bits1(&s->gb);
         sps->offset_for_non_ref_pic= get_se_golomb(&s->gb);
         sps->offset_for_top_to_bottom_field= get_se_golomb(&s->gb);
-        sps->poc_cycle_length= get_ue_golomb(&s->gb);
+        tmp= get_ue_golomb(&s->gb);
+
+        if(tmp >= sizeof(sps->offset_for_ref_frame) / sizeof(sps->offset_for_ref_frame[0])){
+            av_log(h->s.avctx, AV_LOG_ERROR, "poc_cycle_length overflow %u\n", tmp);
+            return -1;
+        }
+        sps->poc_cycle_length= tmp;
 
         for(i=0; i<sps->poc_cycle_length; i++)
             sps->offset_for_ref_frame[i]= get_se_golomb(&s->gb);
-    }
-    if(sps->poc_type > 2){
+    }else if(sps->poc_type != 2){
         av_log(h->s.avctx, AV_LOG_ERROR, "illegal POC type %d\n", sps->poc_type);
         return -1;
     }
 
-    sps->ref_frame_count= get_ue_golomb(&s->gb);
-    if(sps->ref_frame_count > MAX_PICTURE_COUNT-2){
+    tmp= get_ue_golomb(&s->gb);
+    if(tmp > MAX_PICTURE_COUNT-2){
         av_log(h->s.avctx, AV_LOG_ERROR, "too many reference frames\n");
     }
+    sps->ref_frame_count= tmp;
     sps->gaps_in_frame_num_allowed_flag= get_bits1(&s->gb);
-    sps->mb_width= get_ue_golomb(&s->gb) + 1;
-    sps->mb_height= get_ue_golomb(&s->gb) + 1;
-    if((unsigned)sps->mb_width >= INT_MAX/16 || (unsigned)sps->mb_height >= INT_MAX/16 ||
-       avcodec_check_dimensions(NULL, 16*sps->mb_width, 16*sps->mb_height))
+    mb_width= get_ue_golomb(&s->gb) + 1;
+    mb_height= get_ue_golomb(&s->gb) + 1;
+    if(mb_width >= INT_MAX/16 || mb_height >= INT_MAX/16 ||
+       avcodec_check_dimensions(NULL, 16*mb_width, 16*mb_height)){
+        av_log(h->s.avctx, AV_LOG_ERROR, "mb_width/height overflow\n");
         return -1;
+    }
+    sps->mb_width = mb_width;
+    sps->mb_height= mb_height;
 
     sps->frame_mbs_only_flag= get_bits1(&s->gb);
     if(!sps->frame_mbs_only_flag)
@@ -7799,7 +7860,7 @@ static inline int decode_seq_parameter_set(H264Context *h){
 
 static inline int decode_picture_parameter_set(H264Context *h, int bit_length){
     MpegEncContext * const s = &h->s;
-    unsigned int pps_id= get_ue_golomb(&s->gb);
+    unsigned int tmp, pps_id= get_ue_golomb(&s->gb);
     PPS *pps;
 
     if(pps_id>=MAX_PPS_COUNT){
@@ -7808,7 +7869,13 @@ static inline int decode_picture_parameter_set(H264Context *h, int bit_length){
     }
     pps = &h->pps_buffer[pps_id];
 
-    pps->sps_id= get_ue_golomb(&s->gb);
+    tmp= get_ue_golomb(&s->gb);
+    if(tmp>=MAX_SPS_COUNT){
+        av_log(h->s.avctx, AV_LOG_ERROR, "sps_id out of range\n");
+        return -1;
+    }
+    pps->sps_id= tmp;
+
     pps->cabac= get_bits1(&s->gb);
     pps->pic_order_present= get_bits1(&s->gb);
     pps->slice_group_count= get_ue_golomb(&s->gb) + 1;
@@ -7853,6 +7920,7 @@ static inline int decode_picture_parameter_set(H264Context *h, int bit_length){
     pps->ref_count[1]= get_ue_golomb(&s->gb) + 1;
     if(pps->ref_count[0] > 32 || pps->ref_count[1] > 32){
         av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow (pps)\n");
+        pps->ref_count[0]= pps->ref_count[1]= 1;
         return -1;
     }