h264: do not use 422 functions for monochrome

Fixes invalid memory access.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC:libav-stable@libav.org

6 files changed, 30 insertions, 30 deletions

diff --git a/libavcodec/arm/h264dsp_init_arm.c b/libavcodec/arm/h264dsp_init_arm.c
index b4277a514f..761d96dda2 100644
--- a/libavcodec/arm/h264dsp_init_arm.c
+++ b/libavcodec/arm/h264dsp_init_arm.c
@@ -88,7 +88,7 @@ static void ff_h264dsp_init_neon(H264DSPContext *c, const int bit_depth, const i
     c->h264_idct_dc_add     = ff_h264_idct_dc_add_neon;
     c->h264_idct_add16      = ff_h264_idct_add16_neon;
     c->h264_idct_add16intra = ff_h264_idct_add16intra_neon;
-    if (chroma_format_idc == 1)
+    if (chroma_format_idc <= 1)
         c->h264_idct_add8   = ff_h264_idct_add8_neon;
     c->h264_idct8_add       = ff_h264_idct8_add_neon;
     c->h264_idct8_dc_add    = ff_h264_idct8_dc_add_neon;
diff --git a/libavcodec/h264dsp.c b/libavcodec/h264dsp.c
index 1353c1a72f..877302c2aa 100644
--- a/libavcodec/h264dsp.c
+++ b/libavcodec/h264dsp.c
@@ -54,13 +54,13 @@ void ff_h264dsp_init(H264DSPContext *c, const int bit_depth, const int chroma_fo
     c->h264_idct8_dc_add= FUNC(ff_h264_idct8_dc_add, depth);\
     c->h264_idct_add16     = FUNC(ff_h264_idct_add16, depth);\
     c->h264_idct8_add4     = FUNC(ff_h264_idct8_add4, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_idct_add8  = FUNC(ff_h264_idct_add8, depth);\
     else\
         c->h264_idct_add8  = FUNC(ff_h264_idct_add8_422, depth);\
     c->h264_idct_add16intra= FUNC(ff_h264_idct_add16intra, depth);\
     c->h264_luma_dc_dequant_idct= FUNC(ff_h264_luma_dc_dequant_idct, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_chroma_dc_dequant_idct= FUNC(ff_h264_chroma_dc_dequant_idct, depth);\
     else\
         c->h264_chroma_dc_dequant_idct= FUNC(ff_h264_chroma422_dc_dequant_idct, depth);\
@@ -81,20 +81,20 @@ void ff_h264dsp_init(H264DSPContext *c, const int bit_depth, const int chroma_fo
     c->h264_h_loop_filter_luma_intra= FUNC(h264_h_loop_filter_luma_intra, depth);\
     c->h264_h_loop_filter_luma_mbaff_intra= FUNC(h264_h_loop_filter_luma_mbaff_intra, depth);\
     c->h264_v_loop_filter_chroma= FUNC(h264_v_loop_filter_chroma, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_h_loop_filter_chroma= FUNC(h264_h_loop_filter_chroma, depth);\
     else\
         c->h264_h_loop_filter_chroma= FUNC(h264_h_loop_filter_chroma422, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_h_loop_filter_chroma_mbaff= FUNC(h264_h_loop_filter_chroma_mbaff, depth);\
     else\
         c->h264_h_loop_filter_chroma_mbaff= FUNC(h264_h_loop_filter_chroma422_mbaff, depth);\
     c->h264_v_loop_filter_chroma_intra= FUNC(h264_v_loop_filter_chroma_intra, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_h_loop_filter_chroma_intra= FUNC(h264_h_loop_filter_chroma_intra, depth);\
     else\
         c->h264_h_loop_filter_chroma_intra= FUNC(h264_h_loop_filter_chroma422_intra, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_h_loop_filter_chroma_mbaff_intra= FUNC(h264_h_loop_filter_chroma_mbaff_intra, depth);\
     else\
         c->h264_h_loop_filter_chroma_mbaff_intra= FUNC(h264_h_loop_filter_chroma422_mbaff_intra, depth);\
diff --git a/libavcodec/h264pred.c b/libavcodec/h264pred.c
index 94cf9d06d7..d16e2b15d3 100644
--- a/libavcodec/h264pred.c
+++ b/libavcodec/h264pred.c
@@ -471,7 +471,7 @@ void ff_h264_pred_init(H264PredContext *h, int codec_id, const int bit_depth,
     h->pred8x8l[TOP_DC_PRED         ]= FUNCC(pred8x8l_top_dc              , depth);\
     h->pred8x8l[DC_128_PRED         ]= FUNCC(pred8x8l_128_dc              , depth);\
 \
-    if (chroma_format_idc == 1) {\
+    if (chroma_format_idc <= 1) {\
         h->pred8x8[VERT_PRED8x8   ]= FUNCC(pred8x8_vertical               , depth);\
         h->pred8x8[HOR_PRED8x8    ]= FUNCC(pred8x8_horizontal             , depth);\
     } else {\
@@ -479,7 +479,7 @@ void ff_h264_pred_init(H264PredContext *h, int codec_id, const int bit_depth,
         h->pred8x8[HOR_PRED8x8    ]= FUNCC(pred8x16_horizontal            , depth);\
     }\
     if (codec_id != AV_CODEC_ID_VP8) {\
-        if (chroma_format_idc == 1) {\
+        if (chroma_format_idc <= 1) {\
             h->pred8x8[PLANE_PRED8x8]= FUNCC(pred8x8_plane                , depth);\
         } else {\
             h->pred8x8[PLANE_PRED8x8]= FUNCC(pred8x16_plane               , depth);\
@@ -487,7 +487,7 @@ void ff_h264_pred_init(H264PredContext *h, int codec_id, const int bit_depth,
     } else\
         h->pred8x8[PLANE_PRED8x8]= FUNCD(pred8x8_tm_vp8);\
     if(codec_id != AV_CODEC_ID_RV40 && codec_id != AV_CODEC_ID_VP8){\
-        if (chroma_format_idc == 1) {\
+        if (chroma_format_idc <= 1) {\
             h->pred8x8[DC_PRED8x8     ]= FUNCC(pred8x8_dc                     , depth);\
             h->pred8x8[LEFT_DC_PRED8x8]= FUNCC(pred8x8_left_dc                , depth);\
             h->pred8x8[TOP_DC_PRED8x8 ]= FUNCC(pred8x8_top_dc                 , depth);\
@@ -513,7 +513,7 @@ void ff_h264_pred_init(H264PredContext *h, int codec_id, const int bit_depth,
             h->pred8x8[DC_129_PRED8x8]= FUNCC(pred8x8_129_dc              , depth);\
         }\
     }\
-    if (chroma_format_idc == 1) {\
+    if (chroma_format_idc <= 1) {\
         h->pred8x8[DC_128_PRED8x8 ]= FUNCC(pred8x8_128_dc                 , depth);\
     } else {\
         h->pred8x8[DC_128_PRED8x8 ]= FUNCC(pred8x16_128_dc                , depth);\
@@ -547,7 +547,7 @@ void ff_h264_pred_init(H264PredContext *h, int codec_id, const int bit_depth,
     h->pred4x4_add  [ HOR_PRED   ]= FUNCC(pred4x4_horizontal_add          , depth);\
     h->pred8x8l_add [VERT_PRED   ]= FUNCC(pred8x8l_vertical_add           , depth);\
     h->pred8x8l_add [ HOR_PRED   ]= FUNCC(pred8x8l_horizontal_add         , depth);\
-    if (chroma_format_idc == 1) {\
+    if (chroma_format_idc <= 1) {\
     h->pred8x8_add  [VERT_PRED8x8]= FUNCC(pred8x8_vertical_add            , depth);\
     h->pred8x8_add  [ HOR_PRED8x8]= FUNCC(pred8x8_horizontal_add          , depth);\
     } else {\
diff --git a/libavcodec/ppc/h264_altivec.c b/libavcodec/ppc/h264_altivec.c
index 73e2adb776..246ed1c77e 100644
--- a/libavcodec/ppc/h264_altivec.c
+++ b/libavcodec/ppc/h264_altivec.c
@@ -1005,7 +1005,7 @@ void ff_h264dsp_init_ppc(H264DSPContext *c, const int bit_depth, const int chrom
     if (av_get_cpu_flags() & AV_CPU_FLAG_ALTIVEC) {
     if (bit_depth == 8) {
         c->h264_idct_add = ff_h264_idct_add_altivec;
-        if (chroma_format_idc == 1)
+        if (chroma_format_idc <= 1)
             c->h264_idct_add8 = ff_h264_idct_add8_altivec;
         c->h264_idct_add16 = ff_h264_idct_add16_altivec;
         c->h264_idct_add16intra = ff_h264_idct_add16intra_altivec;
diff --git a/libavcodec/x86/h264_intrapred_init.c b/libavcodec/x86/h264_intrapred_init.c
index 454dd3f0e1..e536f36ca1 100644
--- a/libavcodec/x86/h264_intrapred_init.c
+++ b/libavcodec/x86/h264_intrapred_init.c
@@ -187,7 +187,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
         if (EXTERNAL_MMX(mm_flags)) {
             h->pred16x16[VERT_PRED8x8         ] = ff_pred16x16_vertical_8_mmx;
             h->pred16x16[HOR_PRED8x8          ] = ff_pred16x16_horizontal_8_mmx;
-            if (chroma_format_idc == 1) {
+            if (chroma_format_idc <= 1) {
                 h->pred8x8  [VERT_PRED8x8     ] = ff_pred8x8_vertical_8_mmx;
                 h->pred8x8  [HOR_PRED8x8      ] = ff_pred8x8_horizontal_8_mmx;
             }
@@ -196,7 +196,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_8_mmx;
                 h->pred4x4  [TM_VP8_PRED      ] = ff_pred4x4_tm_vp8_8_mmx;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmx;
                 if (codec_id == AV_CODEC_ID_SVQ3) {
                     if (mm_flags & AV_CPU_FLAG_CMOV)
@@ -212,7 +212,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
         if (EXTERNAL_MMXEXT(mm_flags)) {
             h->pred16x16[HOR_PRED8x8            ] = ff_pred16x16_horizontal_8_mmxext;
             h->pred16x16[DC_PRED8x8             ] = ff_pred16x16_dc_8_mmxext;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 h->pred8x8[HOR_PRED8x8          ] = ff_pred8x8_horizontal_8_mmxext;
             h->pred8x8l [TOP_DC_PRED            ] = ff_pred8x8l_top_dc_8_mmxext;
             h->pred8x8l [DC_PRED                ] = ff_pred8x8l_dc_8_mmxext;
@@ -237,7 +237,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred4x4  [HOR_UP_PRED        ] = ff_pred4x4_horizontal_up_8_mmxext;
             }
             if (codec_id == AV_CODEC_ID_SVQ3 || codec_id == AV_CODEC_ID_H264) {
-                if (chroma_format_idc == 1) {
+                if (chroma_format_idc <= 1) {
                     h->pred8x8[TOP_DC_PRED8x8   ] = ff_pred8x8_top_dc_8_mmxext;
                     h->pred8x8[DC_PRED8x8       ] = ff_pred8x8_dc_8_mmxext;
                 }
@@ -249,7 +249,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred4x4  [TM_VP8_PRED        ] = ff_pred4x4_tm_vp8_8_mmxext;
                 h->pred4x4  [VERT_PRED          ] = ff_pred4x4_vertical_vp8_8_mmxext;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmxext;
                 if (codec_id == AV_CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8  ] = ff_pred16x16_plane_svq3_8_mmxext;
@@ -276,7 +276,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred16x16[PLANE_PRED8x8    ] = ff_pred16x16_tm_vp8_8_sse2;
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_8_sse2;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_8_sse2;
                 if (codec_id == AV_CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_sse2;
@@ -291,7 +291,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
         if (EXTERNAL_SSSE3(mm_flags)) {
             h->pred16x16[HOR_PRED8x8          ] = ff_pred16x16_horizontal_8_ssse3;
             h->pred16x16[DC_PRED8x8           ] = ff_pred16x16_dc_8_ssse3;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 h->pred8x8  [HOR_PRED8x8      ] = ff_pred8x8_horizontal_8_ssse3;
             h->pred8x8l [TOP_DC_PRED          ] = ff_pred8x8l_top_dc_8_ssse3;
             h->pred8x8l [DC_PRED              ] = ff_pred8x8l_dc_8_ssse3;
@@ -307,7 +307,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_8_ssse3;
                 h->pred4x4  [TM_VP8_PRED      ] = ff_pred4x4_tm_vp8_8_ssse3;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_8_ssse3;
                 if (codec_id == AV_CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_ssse3;
@@ -323,7 +323,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
             h->pred4x4[DC_PRED             ] = ff_pred4x4_dc_10_mmxext;
             h->pred4x4[HOR_UP_PRED         ] = ff_pred4x4_horizontal_up_10_mmxext;
 
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 h->pred8x8[DC_PRED8x8      ] = ff_pred8x8_dc_10_mmxext;
 
             h->pred8x8l[DC_128_PRED        ] = ff_pred8x8l_128_dc_10_mmxext;
@@ -342,7 +342,7 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
             h->pred4x4[VERT_RIGHT_PRED     ] = ff_pred4x4_vertical_right_10_sse2;
             h->pred4x4[HOR_DOWN_PRED       ] = ff_pred4x4_horizontal_down_10_sse2;
 
-            if (chroma_format_idc == 1) {
+            if (chroma_format_idc <= 1) {
                 h->pred8x8[DC_PRED8x8      ] = ff_pred8x8_dc_10_sse2;
                 h->pred8x8[TOP_DC_PRED8x8  ] = ff_pred8x8_top_dc_10_sse2;
                 h->pred8x8[PLANE_PRED8x8   ] = ff_pred8x8_plane_10_sse2;
diff --git a/libavcodec/x86/h264dsp_init.c b/libavcodec/x86/h264dsp_init.c
index 73d4990899..fdb2e4a40c 100644
--- a/libavcodec/x86/h264dsp_init.c
+++ b/libavcodec/x86/h264dsp_init.c
@@ -212,7 +212,7 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
 {
     int mm_flags = av_get_cpu_flags();
 
-    if (chroma_format_idc == 1 && EXTERNAL_MMXEXT(mm_flags))
+    if (chroma_format_idc <= 1 && EXTERNAL_MMXEXT(mm_flags))
         c->h264_loop_filter_strength = ff_h264_loop_filter_strength_mmxext;
 
     if (bit_depth == 8) {
@@ -224,7 +224,7 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
 
             c->h264_idct_add16 = ff_h264_idct_add16_8_mmx;
             c->h264_idct8_add4 = ff_h264_idct8_add4_8_mmx;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 c->h264_idct_add8 = ff_h264_idct_add8_8_mmx;
             c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmx;
             if (mm_flags & AV_CPU_FLAG_CMOV)
@@ -235,13 +235,13 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
                 c->h264_idct8_dc_add = ff_h264_idct8_dc_add_8_mmxext;
                 c->h264_idct_add16   = ff_h264_idct_add16_8_mmxext;
                 c->h264_idct8_add4   = ff_h264_idct8_add4_8_mmxext;
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     c->h264_idct_add8 = ff_h264_idct_add8_8_mmxext;
                 c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmxext;
 
                 c->h264_v_loop_filter_chroma       = ff_deblock_v_chroma_8_mmxext;
                 c->h264_v_loop_filter_chroma_intra = ff_deblock_v_chroma_intra_8_mmxext;
-                if (chroma_format_idc == 1) {
+                if (chroma_format_idc <= 1) {
                     c->h264_h_loop_filter_chroma       = ff_deblock_h_chroma_8_mmxext;
                     c->h264_h_loop_filter_chroma_intra = ff_deblock_h_chroma_intra_8_mmxext;
                 }
@@ -264,7 +264,7 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
 
                     c->h264_idct_add16 = ff_h264_idct_add16_8_sse2;
                     c->h264_idct8_add4 = ff_h264_idct8_add4_8_sse2;
-                    if (chroma_format_idc == 1)
+                    if (chroma_format_idc <= 1)
                         c->h264_idct_add8 = ff_h264_idct_add8_8_sse2;
                     c->h264_idct_add16intra      = ff_h264_idct_add16intra_8_sse2;
                     c->h264_luma_dc_dequant_idct = ff_h264_luma_dc_dequant_idct_sse2;
@@ -309,7 +309,7 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
                     c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_sse2;
 
                     c->h264_idct_add16 = ff_h264_idct_add16_10_sse2;
-                    if (chroma_format_idc == 1)
+                    if (chroma_format_idc <= 1)
                         c->h264_idct_add8 = ff_h264_idct_add8_10_sse2;
                     c->h264_idct_add16intra = ff_h264_idct_add16intra_10_sse2;
 #if HAVE_ALIGNED_STACK
@@ -349,7 +349,7 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
                     c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_avx;
 
                     c->h264_idct_add16 = ff_h264_idct_add16_10_avx;
-                    if (chroma_format_idc == 1)
+                    if (chroma_format_idc <= 1)
                         c->h264_idct_add8 = ff_h264_idct_add8_10_avx;
                     c->h264_idct_add16intra = ff_h264_idct_add16intra_10_avx;
 #if HAVE_ALIGNED_STACK