h264: do not use 422 functions for monochrome

Fixes invalid memory access.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC:libav-stable@libav.org

6 files changed, 30 insertions, 30 deletions

diff --git a/libavcodec/arm/h264dsp_init_arm.c b/libavcodec/arm/h264dsp_init_arm.c
index 1c331a495d..5d33b90f28 100644
--- a/libavcodec/arm/h264dsp_init_arm.c
+++ b/libavcodec/arm/h264dsp_init_arm.c
@@ -87,7 +87,7 @@ static void ff_h264dsp_init_neon(H264DSPContext *c, const int bit_depth, const i
     c->h264_idct_dc_add     = ff_h264_idct_dc_add_neon;
     c->h264_idct_add16      = ff_h264_idct_add16_neon;
     c->h264_idct_add16intra = ff_h264_idct_add16intra_neon;
-    if (chroma_format_idc == 1)
+    if (chroma_format_idc <= 1)
         c->h264_idct_add8   = ff_h264_idct_add8_neon;
     c->h264_idct8_add       = ff_h264_idct8_add_neon;
     c->h264_idct8_dc_add    = ff_h264_idct8_dc_add_neon;
diff --git a/libavcodec/h264dsp.c b/libavcodec/h264dsp.c
index ba967079fb..4d67f16b51 100644
--- a/libavcodec/h264dsp.c
+++ b/libavcodec/h264dsp.c
@@ -53,13 +53,13 @@ void ff_h264dsp_init(H264DSPContext *c, const int bit_depth, const int chroma_fo
     c->h264_idct8_dc_add= FUNC(ff_h264_idct8_dc_add, depth);\
     c->h264_idct_add16     = FUNC(ff_h264_idct_add16, depth);\
     c->h264_idct8_add4     = FUNC(ff_h264_idct8_add4, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_idct_add8  = FUNC(ff_h264_idct_add8, depth);\
     else\
         c->h264_idct_add8  = FUNC(ff_h264_idct_add8_422, depth);\
     c->h264_idct_add16intra= FUNC(ff_h264_idct_add16intra, depth);\
     c->h264_luma_dc_dequant_idct= FUNC(ff_h264_luma_dc_dequant_idct, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_chroma_dc_dequant_idct= FUNC(ff_h264_chroma_dc_dequant_idct, depth);\
     else\
         c->h264_chroma_dc_dequant_idct= FUNC(ff_h264_chroma422_dc_dequant_idct, depth);\
@@ -80,20 +80,20 @@ void ff_h264dsp_init(H264DSPContext *c, const int bit_depth, const int chroma_fo
     c->h264_h_loop_filter_luma_intra= FUNC(h264_h_loop_filter_luma_intra, depth);\
     c->h264_h_loop_filter_luma_mbaff_intra= FUNC(h264_h_loop_filter_luma_mbaff_intra, depth);\
     c->h264_v_loop_filter_chroma= FUNC(h264_v_loop_filter_chroma, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_h_loop_filter_chroma= FUNC(h264_h_loop_filter_chroma, depth);\
     else\
         c->h264_h_loop_filter_chroma= FUNC(h264_h_loop_filter_chroma422, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_h_loop_filter_chroma_mbaff= FUNC(h264_h_loop_filter_chroma_mbaff, depth);\
     else\
         c->h264_h_loop_filter_chroma_mbaff= FUNC(h264_h_loop_filter_chroma422_mbaff, depth);\
     c->h264_v_loop_filter_chroma_intra= FUNC(h264_v_loop_filter_chroma_intra, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_h_loop_filter_chroma_intra= FUNC(h264_h_loop_filter_chroma_intra, depth);\
     else\
         c->h264_h_loop_filter_chroma_intra= FUNC(h264_h_loop_filter_chroma422_intra, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_h_loop_filter_chroma_mbaff_intra= FUNC(h264_h_loop_filter_chroma_mbaff_intra, depth);\
     else\
         c->h264_h_loop_filter_chroma_mbaff_intra= FUNC(h264_h_loop_filter_chroma422_mbaff_intra, depth);\
diff --git a/libavcodec/h264pred.c b/libavcodec/h264pred.c
index 37a4cf1486..ba6eb054d2 100644
--- a/libavcodec/h264pred.c
+++ b/libavcodec/h264pred.c
@@ -434,7 +434,7 @@ void ff_h264_pred_init(H264PredContext *h, int codec_id, const int bit_depth, co
     h->pred8x8l[TOP_DC_PRED         ]= FUNCC(pred8x8l_top_dc              , depth);\
     h->pred8x8l[DC_128_PRED         ]= FUNCC(pred8x8l_128_dc              , depth);\
 \
-    if (chroma_format_idc == 1) {\
+    if (chroma_format_idc <= 1) {\
         h->pred8x8[VERT_PRED8x8   ]= FUNCC(pred8x8_vertical               , depth);\
         h->pred8x8[HOR_PRED8x8    ]= FUNCC(pred8x8_horizontal             , depth);\
     } else {\
@@ -442,7 +442,7 @@ void ff_h264_pred_init(H264PredContext *h, int codec_id, const int bit_depth, co
         h->pred8x8[HOR_PRED8x8    ]= FUNCC(pred8x16_horizontal            , depth);\
     }\
     if (codec_id != CODEC_ID_VP8) {\
-        if (chroma_format_idc == 1) {\
+        if (chroma_format_idc <= 1) {\
             h->pred8x8[PLANE_PRED8x8]= FUNCC(pred8x8_plane                , depth);\
         } else {\
             h->pred8x8[PLANE_PRED8x8]= FUNCC(pred8x16_plane               , depth);\
@@ -450,7 +450,7 @@ void ff_h264_pred_init(H264PredContext *h, int codec_id, const int bit_depth, co
     } else\
         h->pred8x8[PLANE_PRED8x8]= FUNCD(pred8x8_tm_vp8);\
     if(codec_id != CODEC_ID_RV40 && codec_id != CODEC_ID_VP8){\
-        if (chroma_format_idc == 1) {\
+        if (chroma_format_idc <= 1) {\
             h->pred8x8[DC_PRED8x8     ]= FUNCC(pred8x8_dc                     , depth);\
             h->pred8x8[LEFT_DC_PRED8x8]= FUNCC(pred8x8_left_dc                , depth);\
             h->pred8x8[TOP_DC_PRED8x8 ]= FUNCC(pred8x8_top_dc                 , depth);\
@@ -476,7 +476,7 @@ void ff_h264_pred_init(H264PredContext *h, int codec_id, const int bit_depth, co
             h->pred8x8[DC_129_PRED8x8]= FUNCC(pred8x8_129_dc              , depth);\
         }\
     }\
-    if (chroma_format_idc == 1) {\
+    if (chroma_format_idc <= 1) {\
         h->pred8x8[DC_128_PRED8x8 ]= FUNCC(pred8x8_128_dc                 , depth);\
     } else {\
         h->pred8x8[DC_128_PRED8x8 ]= FUNCC(pred8x16_128_dc                , depth);\
@@ -510,7 +510,7 @@ void ff_h264_pred_init(H264PredContext *h, int codec_id, const int bit_depth, co
     h->pred4x4_add  [ HOR_PRED   ]= FUNCC(pred4x4_horizontal_add          , depth);\
     h->pred8x8l_add [VERT_PRED   ]= FUNCC(pred8x8l_vertical_add           , depth);\
     h->pred8x8l_add [ HOR_PRED   ]= FUNCC(pred8x8l_horizontal_add         , depth);\
-    if (chroma_format_idc == 1) {\
+    if (chroma_format_idc <= 1) {\
     h->pred8x8_add  [VERT_PRED8x8]= FUNCC(pred8x8_vertical_add            , depth);\
     h->pred8x8_add  [ HOR_PRED8x8]= FUNCC(pred8x8_horizontal_add          , depth);\
     } else {\
diff --git a/libavcodec/ppc/h264_altivec.c b/libavcodec/ppc/h264_altivec.c
index c8baee456e..f66aed1901 100644
--- a/libavcodec/ppc/h264_altivec.c
+++ b/libavcodec/ppc/h264_altivec.c
@@ -1004,7 +1004,7 @@ void ff_h264dsp_init_ppc(H264DSPContext *c, const int bit_depth, const int chrom
     if (av_get_cpu_flags() & AV_CPU_FLAG_ALTIVEC) {
     if (bit_depth == 8) {
         c->h264_idct_add = ff_h264_idct_add_altivec;
-        if (chroma_format_idc == 1)
+        if (chroma_format_idc <= 1)
             c->h264_idct_add8 = ff_h264_idct_add8_altivec;
         c->h264_idct_add16 = ff_h264_idct_add16_altivec;
         c->h264_idct_add16intra = ff_h264_idct_add16intra_altivec;
diff --git a/libavcodec/x86/h264_intrapred_init.c b/libavcodec/x86/h264_intrapred_init.c
index 41e611ecd1..223dbde2ae 100644
--- a/libavcodec/x86/h264_intrapred_init.c
+++ b/libavcodec/x86/h264_intrapred_init.c
@@ -176,7 +176,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
         if (mm_flags & AV_CPU_FLAG_MMX) {
             h->pred16x16[VERT_PRED8x8         ] = ff_pred16x16_vertical_mmx;
             h->pred16x16[HOR_PRED8x8          ] = ff_pred16x16_horizontal_mmx;
-            if (chroma_format_idc == 1) {
+            if (chroma_format_idc <= 1) {
                 h->pred8x8  [VERT_PRED8x8     ] = ff_pred8x8_vertical_mmx;
                 h->pred8x8  [HOR_PRED8x8      ] = ff_pred8x8_horizontal_mmx;
             }
@@ -185,7 +185,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_mmx;
                 h->pred4x4  [TM_VP8_PRED      ] = ff_pred4x4_tm_vp8_mmx;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_mmx;
                 if (codec_id == CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_mmx;
@@ -200,7 +200,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
         if (mm_flags & AV_CPU_FLAG_MMX2) {
             h->pred16x16[HOR_PRED8x8            ] = ff_pred16x16_horizontal_mmxext;
             h->pred16x16[DC_PRED8x8             ] = ff_pred16x16_dc_mmxext;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 h->pred8x8[HOR_PRED8x8          ] = ff_pred8x8_horizontal_mmxext;
             h->pred8x8l [TOP_DC_PRED            ] = ff_pred8x8l_top_dc_mmxext;
             h->pred8x8l [DC_PRED                ] = ff_pred8x8l_dc_mmxext;
@@ -225,7 +225,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred4x4  [HOR_UP_PRED        ] = ff_pred4x4_horizontal_up_mmxext;
             }
             if (codec_id == CODEC_ID_SVQ3 || codec_id == CODEC_ID_H264) {
-                if (chroma_format_idc == 1) {
+                if (chroma_format_idc <= 1) {
                     h->pred8x8[TOP_DC_PRED8x8   ] = ff_pred8x8_top_dc_mmxext;
                     h->pred8x8[DC_PRED8x8       ] = ff_pred8x8_dc_mmxext;
                 }
@@ -237,7 +237,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred4x4  [TM_VP8_PRED        ] = ff_pred4x4_tm_vp8_mmxext;
                 h->pred4x4  [VERT_PRED          ] = ff_pred4x4_vertical_vp8_mmxext;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_mmx2;
                 if (codec_id == CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8  ] = ff_pred16x16_plane_svq3_mmx2;
@@ -264,7 +264,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred16x16[PLANE_PRED8x8    ] = ff_pred16x16_tm_vp8_sse2;
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_sse2;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_sse2;
                 if (codec_id == CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_sse2;
@@ -279,7 +279,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
         if (mm_flags & AV_CPU_FLAG_SSSE3) {
             h->pred16x16[HOR_PRED8x8          ] = ff_pred16x16_horizontal_ssse3;
             h->pred16x16[DC_PRED8x8           ] = ff_pred16x16_dc_ssse3;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 h->pred8x8  [HOR_PRED8x8      ] = ff_pred8x8_horizontal_ssse3;
             h->pred8x8l [TOP_DC_PRED          ] = ff_pred8x8l_top_dc_ssse3;
             h->pred8x8l [DC_PRED              ] = ff_pred8x8l_dc_ssse3;
@@ -295,7 +295,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_ssse3;
                 h->pred4x4  [TM_VP8_PRED      ] = ff_pred4x4_tm_vp8_ssse3;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_ssse3;
                 if (codec_id == CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_ssse3;
@@ -311,7 +311,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
             h->pred4x4[DC_PRED             ] = ff_pred4x4_dc_10_mmxext;
             h->pred4x4[HOR_UP_PRED         ] = ff_pred4x4_horizontal_up_10_mmxext;
 
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 h->pred8x8[DC_PRED8x8      ] = ff_pred8x8_dc_10_mmxext;
 
             h->pred8x8l[DC_128_PRED        ] = ff_pred8x8l_128_dc_10_mmxext;
@@ -330,7 +330,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
             h->pred4x4[VERT_RIGHT_PRED     ] = ff_pred4x4_vertical_right_10_sse2;
             h->pred4x4[HOR_DOWN_PRED       ] = ff_pred4x4_horizontal_down_10_sse2;
 
-            if (chroma_format_idc == 1) {
+            if (chroma_format_idc <= 1) {
                 h->pred8x8[DC_PRED8x8      ] = ff_pred8x8_dc_10_sse2;
                 h->pred8x8[TOP_DC_PRED8x8  ] = ff_pred8x8_top_dc_10_sse2;
                 h->pred8x8[PLANE_PRED8x8   ] = ff_pred8x8_plane_10_sse2;
diff --git a/libavcodec/x86/h264dsp_mmx.c b/libavcodec/x86/h264dsp_mmx.c
index dcd918013c..f5ae4dc055 100644
--- a/libavcodec/x86/h264dsp_mmx.c
+++ b/libavcodec/x86/h264dsp_mmx.c
@@ -344,7 +344,7 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, const int chrom
 {
     int mm_flags = av_get_cpu_flags();
 
-    if (chroma_format_idc == 1 && mm_flags & AV_CPU_FLAG_MMX2) {
+    if (chroma_format_idc <= 1 && mm_flags & AV_CPU_FLAG_MMX2) {
         c->h264_loop_filter_strength= h264_loop_filter_strength_mmx2;
     }
 
@@ -358,7 +358,7 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, const int chrom
 
         c->h264_idct_add16          = ff_h264_idct_add16_8_mmx;
         c->h264_idct8_add4          = ff_h264_idct8_add4_8_mmx;
-        if (chroma_format_idc == 1)
+        if (chroma_format_idc <= 1)
             c->h264_idct_add8       = ff_h264_idct_add8_8_mmx;
         c->h264_idct_add16intra     = ff_h264_idct_add16intra_8_mmx;
         c->h264_luma_dc_dequant_idct= ff_h264_luma_dc_dequant_idct_mmx;
@@ -368,13 +368,13 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, const int chrom
             c->h264_idct8_dc_add   = ff_h264_idct8_dc_add_8_mmx2;
             c->h264_idct_add16     = ff_h264_idct_add16_8_mmx2;
             c->h264_idct8_add4     = ff_h264_idct8_add4_8_mmx2;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 c->h264_idct_add8  = ff_h264_idct_add8_8_mmx2;
             c->h264_idct_add16intra= ff_h264_idct_add16intra_8_mmx2;
 
             c->h264_v_loop_filter_chroma= ff_deblock_v_chroma_8_mmxext;
             c->h264_v_loop_filter_chroma_intra= ff_deblock_v_chroma_intra_8_mmxext;
-            if (chroma_format_idc == 1) {
+            if (chroma_format_idc <= 1) {
                 c->h264_h_loop_filter_chroma= ff_deblock_h_chroma_8_mmxext;
                 c->h264_h_loop_filter_chroma_intra= ff_deblock_h_chroma_intra_8_mmxext;
             }
@@ -397,7 +397,7 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, const int chrom
 
                 c->h264_idct_add16          = ff_h264_idct_add16_8_sse2;
                 c->h264_idct8_add4          = ff_h264_idct8_add4_8_sse2;
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     c->h264_idct_add8       = ff_h264_idct_add8_8_sse2;
                 c->h264_idct_add16intra     = ff_h264_idct_add16intra_8_sse2;
                 c->h264_luma_dc_dequant_idct= ff_h264_luma_dc_dequant_idct_sse2;
@@ -448,7 +448,7 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, const int chrom
                 c->h264_idct8_dc_add   = ff_h264_idct8_dc_add_10_sse2;
 
                 c->h264_idct_add16     = ff_h264_idct_add16_10_sse2;
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     c->h264_idct_add8  = ff_h264_idct_add8_10_sse2;
                 c->h264_idct_add16intra= ff_h264_idct_add16intra_10_sse2;
 #if HAVE_ALIGNED_STACK
@@ -489,7 +489,7 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, const int chrom
                 c->h264_idct8_dc_add   = ff_h264_idct8_dc_add_10_avx;
 
                 c->h264_idct_add16     = ff_h264_idct_add16_10_avx;
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     c->h264_idct_add8  = ff_h264_idct_add8_10_avx;
                 c->h264_idct_add16intra= ff_h264_idct_add16intra_10_avx;
 #if HAVE_ALIGNED_STACK