h264: do not use 422 functions for monochrome

Fixes invalid memory access.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC:libav-stable@libav.org

6 files changed, 30 insertions, 30 deletions

diff --git a/libavcodec/arm/h264dsp_init_arm.c b/libavcodec/arm/h264dsp_init_arm.c
index b206a1b3ba..c1398a660b 100644
--- a/libavcodec/arm/h264dsp_init_arm.c
+++ b/libavcodec/arm/h264dsp_init_arm.c
@@ -91,7 +91,7 @@ static av_cold void h264dsp_init_neon(H264DSPContext *c, const int bit_depth,
     c->h264_idct_dc_add     = ff_h264_idct_dc_add_neon;
     c->h264_idct_add16      = ff_h264_idct_add16_neon;
     c->h264_idct_add16intra = ff_h264_idct_add16intra_neon;
-    if (chroma_format_idc == 1)
+    if (chroma_format_idc <= 1)
         c->h264_idct_add8   = ff_h264_idct_add8_neon;
     c->h264_idct8_add       = ff_h264_idct8_add_neon;
     c->h264_idct8_dc_add    = ff_h264_idct8_dc_add_neon;
diff --git a/libavcodec/h264dsp.c b/libavcodec/h264dsp.c
index a901dbb9e1..0824118eb1 100644
--- a/libavcodec/h264dsp.c
+++ b/libavcodec/h264dsp.c
@@ -104,13 +104,13 @@ av_cold void ff_h264dsp_init(H264DSPContext *c, const int bit_depth,
     c->h264_idct8_dc_add= FUNC(ff_h264_idct8_dc_add, depth);\
     c->h264_idct_add16     = FUNC(ff_h264_idct_add16, depth);\
     c->h264_idct8_add4     = FUNC(ff_h264_idct8_add4, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_idct_add8  = FUNC(ff_h264_idct_add8, depth);\
     else\
         c->h264_idct_add8  = FUNC(ff_h264_idct_add8_422, depth);\
     c->h264_idct_add16intra= FUNC(ff_h264_idct_add16intra, depth);\
     c->h264_luma_dc_dequant_idct= FUNC(ff_h264_luma_dc_dequant_idct, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_chroma_dc_dequant_idct= FUNC(ff_h264_chroma_dc_dequant_idct, depth);\
     else\
         c->h264_chroma_dc_dequant_idct= FUNC(ff_h264_chroma422_dc_dequant_idct, depth);\
@@ -131,20 +131,20 @@ av_cold void ff_h264dsp_init(H264DSPContext *c, const int bit_depth,
     c->h264_h_loop_filter_luma_intra= FUNC(h264_h_loop_filter_luma_intra, depth);\
     c->h264_h_loop_filter_luma_mbaff_intra= FUNC(h264_h_loop_filter_luma_mbaff_intra, depth);\
     c->h264_v_loop_filter_chroma= FUNC(h264_v_loop_filter_chroma, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_h_loop_filter_chroma= FUNC(h264_h_loop_filter_chroma, depth);\
     else\
         c->h264_h_loop_filter_chroma= FUNC(h264_h_loop_filter_chroma422, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_h_loop_filter_chroma_mbaff= FUNC(h264_h_loop_filter_chroma_mbaff, depth);\
     else\
         c->h264_h_loop_filter_chroma_mbaff= FUNC(h264_h_loop_filter_chroma422_mbaff, depth);\
     c->h264_v_loop_filter_chroma_intra= FUNC(h264_v_loop_filter_chroma_intra, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_h_loop_filter_chroma_intra= FUNC(h264_h_loop_filter_chroma_intra, depth);\
     else\
         c->h264_h_loop_filter_chroma_intra= FUNC(h264_h_loop_filter_chroma422_intra, depth);\
-    if (chroma_format_idc == 1)\
+    if (chroma_format_idc <= 1)\
         c->h264_h_loop_filter_chroma_mbaff_intra= FUNC(h264_h_loop_filter_chroma_mbaff_intra, depth);\
     else\
         c->h264_h_loop_filter_chroma_mbaff_intra= FUNC(h264_h_loop_filter_chroma422_mbaff_intra, depth);\
diff --git a/libavcodec/h264pred.c b/libavcodec/h264pred.c
index 2c991fd414..0d47897dff 100644
--- a/libavcodec/h264pred.c
+++ b/libavcodec/h264pred.c
@@ -472,7 +472,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id,
     h->pred8x8l[TOP_DC_PRED         ]= FUNCC(pred8x8l_top_dc              , depth);\
     h->pred8x8l[DC_128_PRED         ]= FUNCC(pred8x8l_128_dc              , depth);\
 \
-    if (chroma_format_idc == 1) {\
+    if (chroma_format_idc <= 1) {\
         h->pred8x8[VERT_PRED8x8   ]= FUNCC(pred8x8_vertical               , depth);\
         h->pred8x8[HOR_PRED8x8    ]= FUNCC(pred8x8_horizontal             , depth);\
     } else {\
@@ -480,7 +480,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id,
         h->pred8x8[HOR_PRED8x8    ]= FUNCC(pred8x16_horizontal            , depth);\
     }\
     if (codec_id != AV_CODEC_ID_VP8) {\
-        if (chroma_format_idc == 1) {\
+        if (chroma_format_idc <= 1) {\
             h->pred8x8[PLANE_PRED8x8]= FUNCC(pred8x8_plane                , depth);\
         } else {\
             h->pred8x8[PLANE_PRED8x8]= FUNCC(pred8x16_plane               , depth);\
@@ -488,7 +488,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id,
     } else\
         h->pred8x8[PLANE_PRED8x8]= FUNCD(pred8x8_tm_vp8);\
     if(codec_id != AV_CODEC_ID_RV40 && codec_id != AV_CODEC_ID_VP8){\
-        if (chroma_format_idc == 1) {\
+        if (chroma_format_idc <= 1) {\
             h->pred8x8[DC_PRED8x8     ]= FUNCC(pred8x8_dc                     , depth);\
             h->pred8x8[LEFT_DC_PRED8x8]= FUNCC(pred8x8_left_dc                , depth);\
             h->pred8x8[TOP_DC_PRED8x8 ]= FUNCC(pred8x8_top_dc                 , depth);\
@@ -514,7 +514,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id,
             h->pred8x8[DC_129_PRED8x8]= FUNCC(pred8x8_129_dc              , depth);\
         }\
     }\
-    if (chroma_format_idc == 1) {\
+    if (chroma_format_idc <= 1) {\
         h->pred8x8[DC_128_PRED8x8 ]= FUNCC(pred8x8_128_dc                 , depth);\
     } else {\
         h->pred8x8[DC_128_PRED8x8 ]= FUNCC(pred8x16_128_dc                , depth);\
@@ -548,7 +548,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id,
     h->pred4x4_add  [ HOR_PRED   ]= FUNCC(pred4x4_horizontal_add          , depth);\
     h->pred8x8l_add [VERT_PRED   ]= FUNCC(pred8x8l_vertical_add           , depth);\
     h->pred8x8l_add [ HOR_PRED   ]= FUNCC(pred8x8l_horizontal_add         , depth);\
-    if (chroma_format_idc == 1) {\
+    if (chroma_format_idc <= 1) {\
     h->pred8x8_add  [VERT_PRED8x8]= FUNCC(pred8x8_vertical_add            , depth);\
     h->pred8x8_add  [ HOR_PRED8x8]= FUNCC(pred8x8_horizontal_add          , depth);\
     } else {\
diff --git a/libavcodec/ppc/h264dsp.c b/libavcodec/ppc/h264dsp.c
index df298dd21f..93ad43622a 100644
--- a/libavcodec/ppc/h264dsp.c
+++ b/libavcodec/ppc/h264dsp.c
@@ -750,7 +750,7 @@ av_cold void ff_h264dsp_init_ppc(H264DSPContext *c, const int bit_depth,
 
     if (bit_depth == 8) {
         c->h264_idct_add = h264_idct_add_altivec;
-        if (chroma_format_idc == 1)
+        if (chroma_format_idc <= 1)
             c->h264_idct_add8 = h264_idct_add8_altivec;
         c->h264_idct_add16      = h264_idct_add16_altivec;
         c->h264_idct_add16intra = h264_idct_add16intra_altivec;
diff --git a/libavcodec/x86/h264_intrapred_init.c b/libavcodec/x86/h264_intrapred_init.c
index 6dd98aab72..f934256706 100644
--- a/libavcodec/x86/h264_intrapred_init.c
+++ b/libavcodec/x86/h264_intrapred_init.c
@@ -191,7 +191,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
         if (EXTERNAL_MMX(cpu_flags)) {
             h->pred16x16[VERT_PRED8x8         ] = ff_pred16x16_vertical_8_mmx;
             h->pred16x16[HOR_PRED8x8          ] = ff_pred16x16_horizontal_8_mmx;
-            if (chroma_format_idc == 1) {
+            if (chroma_format_idc <= 1) {
                 h->pred8x8  [VERT_PRED8x8     ] = ff_pred8x8_vertical_8_mmx;
                 h->pred8x8  [HOR_PRED8x8      ] = ff_pred8x8_horizontal_8_mmx;
             }
@@ -200,7 +200,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_8_mmx;
                 h->pred4x4  [TM_VP8_PRED      ] = ff_pred4x4_tm_vp8_8_mmx;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmx;
                 if (codec_id == AV_CODEC_ID_SVQ3) {
                     if (cpu_flags & AV_CPU_FLAG_CMOV)
@@ -216,7 +216,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
         if (EXTERNAL_MMXEXT(cpu_flags)) {
             h->pred16x16[HOR_PRED8x8            ] = ff_pred16x16_horizontal_8_mmxext;
             h->pred16x16[DC_PRED8x8             ] = ff_pred16x16_dc_8_mmxext;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 h->pred8x8[HOR_PRED8x8          ] = ff_pred8x8_horizontal_8_mmxext;
             h->pred8x8l [TOP_DC_PRED            ] = ff_pred8x8l_top_dc_8_mmxext;
             h->pred8x8l [DC_PRED                ] = ff_pred8x8l_dc_8_mmxext;
@@ -241,7 +241,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
                 h->pred4x4  [HOR_UP_PRED        ] = ff_pred4x4_horizontal_up_8_mmxext;
             }
             if (codec_id == AV_CODEC_ID_SVQ3 || codec_id == AV_CODEC_ID_H264) {
-                if (chroma_format_idc == 1) {
+                if (chroma_format_idc <= 1) {
                     h->pred8x8[TOP_DC_PRED8x8   ] = ff_pred8x8_top_dc_8_mmxext;
                     h->pred8x8[DC_PRED8x8       ] = ff_pred8x8_dc_8_mmxext;
                 }
@@ -253,7 +253,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
                 h->pred4x4  [TM_VP8_PRED        ] = ff_pred4x4_tm_vp8_8_mmxext;
                 h->pred4x4  [VERT_PRED          ] = ff_pred4x4_vertical_vp8_8_mmxext;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmxext;
                 if (codec_id == AV_CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8  ] = ff_pred16x16_plane_svq3_8_mmxext;
@@ -280,7 +280,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
                 h->pred16x16[PLANE_PRED8x8    ] = ff_pred16x16_tm_vp8_8_sse2;
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_8_sse2;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_8_sse2;
                 if (codec_id == AV_CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_sse2;
@@ -295,7 +295,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
         if (EXTERNAL_SSSE3(cpu_flags)) {
             h->pred16x16[HOR_PRED8x8          ] = ff_pred16x16_horizontal_8_ssse3;
             h->pred16x16[DC_PRED8x8           ] = ff_pred16x16_dc_8_ssse3;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 h->pred8x8  [HOR_PRED8x8      ] = ff_pred8x8_horizontal_8_ssse3;
             h->pred8x8l [TOP_DC_PRED          ] = ff_pred8x8l_top_dc_8_ssse3;
             h->pred8x8l [DC_PRED              ] = ff_pred8x8l_dc_8_ssse3;
@@ -311,7 +311,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_8_ssse3;
                 h->pred4x4  [TM_VP8_PRED      ] = ff_pred4x4_tm_vp8_8_ssse3;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_8_ssse3;
                 if (codec_id == AV_CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_ssse3;
@@ -327,7 +327,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
             h->pred4x4[DC_PRED             ] = ff_pred4x4_dc_10_mmxext;
             h->pred4x4[HOR_UP_PRED         ] = ff_pred4x4_horizontal_up_10_mmxext;
 
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 h->pred8x8[DC_PRED8x8      ] = ff_pred8x8_dc_10_mmxext;
 
             h->pred8x8l[DC_128_PRED        ] = ff_pred8x8l_128_dc_10_mmxext;
@@ -346,7 +346,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
             h->pred4x4[VERT_RIGHT_PRED     ] = ff_pred4x4_vertical_right_10_sse2;
             h->pred4x4[HOR_DOWN_PRED       ] = ff_pred4x4_horizontal_down_10_sse2;
 
-            if (chroma_format_idc == 1) {
+            if (chroma_format_idc <= 1) {
                 h->pred8x8[DC_PRED8x8      ] = ff_pred8x8_dc_10_sse2;
                 h->pred8x8[TOP_DC_PRED8x8  ] = ff_pred8x8_top_dc_10_sse2;
                 h->pred8x8[PLANE_PRED8x8   ] = ff_pred8x8_plane_10_sse2;
diff --git a/libavcodec/x86/h264dsp_init.c b/libavcodec/x86/h264dsp_init.c
index 4164b833db..e9d93e0af9 100644
--- a/libavcodec/x86/h264dsp_init.c
+++ b/libavcodec/x86/h264dsp_init.c
@@ -213,7 +213,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
 {
     int cpu_flags = av_get_cpu_flags();
 
-    if (chroma_format_idc == 1 && EXTERNAL_MMXEXT(cpu_flags))
+    if (chroma_format_idc <= 1 && EXTERNAL_MMXEXT(cpu_flags))
         c->h264_loop_filter_strength = ff_h264_loop_filter_strength_mmxext;
 
     if (bit_depth == 8) {
@@ -225,7 +225,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
 
             c->h264_idct_add16 = ff_h264_idct_add16_8_mmx;
             c->h264_idct8_add4 = ff_h264_idct8_add4_8_mmx;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 c->h264_idct_add8 = ff_h264_idct_add8_8_mmx;
             c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmx;
             if (cpu_flags & AV_CPU_FLAG_CMOV)
@@ -236,13 +236,13 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
             c->h264_idct8_dc_add = ff_h264_idct8_dc_add_8_mmxext;
             c->h264_idct_add16   = ff_h264_idct_add16_8_mmxext;
             c->h264_idct8_add4   = ff_h264_idct8_add4_8_mmxext;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 c->h264_idct_add8 = ff_h264_idct_add8_8_mmxext;
             c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmxext;
 
             c->h264_v_loop_filter_chroma       = ff_deblock_v_chroma_8_mmxext;
             c->h264_v_loop_filter_chroma_intra = ff_deblock_v_chroma_intra_8_mmxext;
-            if (chroma_format_idc == 1) {
+            if (chroma_format_idc <= 1) {
                 c->h264_h_loop_filter_chroma       = ff_deblock_h_chroma_8_mmxext;
                 c->h264_h_loop_filter_chroma_intra = ff_deblock_h_chroma_intra_8_mmxext;
             }
@@ -265,7 +265,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
 
             c->h264_idct_add16 = ff_h264_idct_add16_8_sse2;
             c->h264_idct8_add4 = ff_h264_idct8_add4_8_sse2;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 c->h264_idct_add8 = ff_h264_idct_add8_8_sse2;
             c->h264_idct_add16intra      = ff_h264_idct_add16intra_8_sse2;
             c->h264_luma_dc_dequant_idct = ff_h264_luma_dc_dequant_idct_sse2;
@@ -308,7 +308,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
             c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_sse2;
 
             c->h264_idct_add16 = ff_h264_idct_add16_10_sse2;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 c->h264_idct_add8 = ff_h264_idct_add8_10_sse2;
             c->h264_idct_add16intra = ff_h264_idct_add16intra_10_sse2;
 #if HAVE_ALIGNED_STACK
@@ -348,7 +348,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
             c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_avx;
 
             c->h264_idct_add16 = ff_h264_idct_add16_10_avx;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 c->h264_idct_add8 = ff_h264_idct_add8_10_avx;
             c->h264_idct_add16intra = ff_h264_idct_add16intra_10_avx;
 #if HAVE_ALIGNED_STACK