h264dec: h264: 4:2:2 intra decoding

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

2 files changed, 45 insertions, 25 deletions

diff --git a/libavcodec/x86/h264_intrapred_init.c b/libavcodec/x86/h264_intrapred_init.c
index 7220dd75c1..cdf7d55d96 100644
--- a/libavcodec/x86/h264_intrapred_init.c
+++ b/libavcodec/x86/h264_intrapred_init.c
@@ -167,7 +167,7 @@ void ff_pred4x4_tm_vp8_mmxext      (uint8_t *src, const uint8_t *topright, int s
 void ff_pred4x4_tm_vp8_ssse3       (uint8_t *src, const uint8_t *topright, int stride);
 void ff_pred4x4_vertical_vp8_mmxext(uint8_t *src, const uint8_t *topright, int stride);
 
-void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth)
+void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth, const int chroma_format_idc)
 {
     int mm_flags = av_get_cpu_flags();
 
@@ -176,14 +176,17 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
         if (mm_flags & AV_CPU_FLAG_MMX) {
             h->pred16x16[VERT_PRED8x8         ] = ff_pred16x16_vertical_mmx;
             h->pred16x16[HOR_PRED8x8          ] = ff_pred16x16_horizontal_mmx;
-            h->pred8x8  [VERT_PRED8x8         ] = ff_pred8x8_vertical_mmx;
-            h->pred8x8  [HOR_PRED8x8          ] = ff_pred8x8_horizontal_mmx;
+            if (chroma_format_idc == 1) {
+                h->pred8x8  [VERT_PRED8x8     ] = ff_pred8x8_vertical_mmx;
+                h->pred8x8  [HOR_PRED8x8      ] = ff_pred8x8_horizontal_mmx;
+            }
             if (codec_id == CODEC_ID_VP8) {
                 h->pred16x16[PLANE_PRED8x8    ] = ff_pred16x16_tm_vp8_mmx;
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_mmx;
                 h->pred4x4  [TM_VP8_PRED      ] = ff_pred4x4_tm_vp8_mmx;
             } else {
-                h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_mmx;
+                if (chroma_format_idc == 1)
+                    h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_mmx;
                 if (codec_id == CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_mmx;
                 } else if (codec_id == CODEC_ID_RV40) {
@@ -197,7 +200,8 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
         if (mm_flags & AV_CPU_FLAG_MMX2) {
             h->pred16x16[HOR_PRED8x8            ] = ff_pred16x16_horizontal_mmxext;
             h->pred16x16[DC_PRED8x8             ] = ff_pred16x16_dc_mmxext;
-            h->pred8x8  [HOR_PRED8x8            ] = ff_pred8x8_horizontal_mmxext;
+            if (chroma_format_idc == 1)
+                h->pred8x8[HOR_PRED8x8          ] = ff_pred8x8_horizontal_mmxext;
             h->pred8x8l [TOP_DC_PRED            ] = ff_pred8x8l_top_dc_mmxext;
             h->pred8x8l [DC_PRED                ] = ff_pred8x8l_dc_mmxext;
             h->pred8x8l [HOR_PRED               ] = ff_pred8x8l_horizontal_mmxext;
@@ -221,8 +225,10 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred4x4  [HOR_UP_PRED        ] = ff_pred4x4_horizontal_up_mmxext;
             }
             if (codec_id == CODEC_ID_SVQ3 || codec_id == CODEC_ID_H264) {
-                h->pred8x8  [TOP_DC_PRED8x8     ] = ff_pred8x8_top_dc_mmxext;
-                h->pred8x8  [DC_PRED8x8         ] = ff_pred8x8_dc_mmxext;
+                if (chroma_format_idc == 1) {
+                    h->pred8x8[TOP_DC_PRED8x8   ] = ff_pred8x8_top_dc_mmxext;
+                    h->pred8x8[DC_PRED8x8       ] = ff_pred8x8_dc_mmxext;
+                }
             }
             if (codec_id == CODEC_ID_VP8) {
                 h->pred16x16[PLANE_PRED8x8      ] = ff_pred16x16_tm_vp8_mmxext;
@@ -231,7 +237,8 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred4x4  [TM_VP8_PRED        ] = ff_pred4x4_tm_vp8_mmxext;
                 h->pred4x4  [VERT_PRED          ] = ff_pred4x4_vertical_vp8_mmxext;
             } else {
-                h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_mmx2;
+                if (chroma_format_idc == 1)
+                    h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_mmx2;
                 if (codec_id == CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8  ] = ff_pred16x16_plane_svq3_mmx2;
                 } else if (codec_id == CODEC_ID_RV40) {
@@ -257,7 +264,8 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred16x16[PLANE_PRED8x8    ] = ff_pred16x16_tm_vp8_sse2;
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_sse2;
             } else {
-                h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_plane_sse2;
+                if (chroma_format_idc == 1)
+                    h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_sse2;
                 if (codec_id == CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_sse2;
                 } else if (codec_id == CODEC_ID_RV40) {
@@ -271,7 +279,8 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
         if (mm_flags & AV_CPU_FLAG_SSSE3) {
             h->pred16x16[HOR_PRED8x8          ] = ff_pred16x16_horizontal_ssse3;
             h->pred16x16[DC_PRED8x8           ] = ff_pred16x16_dc_ssse3;
-            h->pred8x8  [HOR_PRED8x8          ] = ff_pred8x8_horizontal_ssse3;
+            if (chroma_format_idc == 1)
+                h->pred8x8  [HOR_PRED8x8      ] = ff_pred8x8_horizontal_ssse3;
             h->pred8x8l [TOP_DC_PRED          ] = ff_pred8x8l_top_dc_ssse3;
             h->pred8x8l [DC_PRED              ] = ff_pred8x8l_dc_ssse3;
             h->pred8x8l [HOR_PRED             ] = ff_pred8x8l_horizontal_ssse3;
@@ -286,7 +295,8 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_ssse3;
                 h->pred4x4  [TM_VP8_PRED      ] = ff_pred4x4_tm_vp8_ssse3;
             } else {
-                h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_ssse3;
+                if (chroma_format_idc == 1)
+                    h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_ssse3;
                 if (codec_id == CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_ssse3;
                 } else if (codec_id == CODEC_ID_RV40) {
@@ -301,7 +311,8 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
             h->pred4x4[DC_PRED             ] = ff_pred4x4_dc_10_mmxext;
             h->pred4x4[HOR_UP_PRED         ] = ff_pred4x4_horizontal_up_10_mmxext;
 
-            h->pred8x8[DC_PRED8x8          ] = ff_pred8x8_dc_10_mmxext;
+            if (chroma_format_idc == 1)
+                h->pred8x8[DC_PRED8x8      ] = ff_pred8x8_dc_10_mmxext;
 
             h->pred8x8l[DC_128_PRED        ] = ff_pred8x8l_128_dc_10_mmxext;
 
@@ -319,11 +330,13 @@ void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, const int bit_depth
             h->pred4x4[VERT_RIGHT_PRED     ] = ff_pred4x4_vertical_right_10_sse2;
             h->pred4x4[HOR_DOWN_PRED       ] = ff_pred4x4_horizontal_down_10_sse2;
 
-            h->pred8x8[DC_PRED8x8          ] = ff_pred8x8_dc_10_sse2;
-            h->pred8x8[TOP_DC_PRED8x8      ] = ff_pred8x8_top_dc_10_sse2;
-            h->pred8x8[PLANE_PRED8x8       ] = ff_pred8x8_plane_10_sse2;
-            h->pred8x8[VERT_PRED8x8        ] = ff_pred8x8_vertical_10_sse2;
-            h->pred8x8[HOR_PRED8x8         ] = ff_pred8x8_horizontal_10_sse2;
+            if (chroma_format_idc == 1) {
+                h->pred8x8[DC_PRED8x8      ] = ff_pred8x8_dc_10_sse2;
+                h->pred8x8[TOP_DC_PRED8x8  ] = ff_pred8x8_top_dc_10_sse2;
+                h->pred8x8[PLANE_PRED8x8   ] = ff_pred8x8_plane_10_sse2;
+                h->pred8x8[VERT_PRED8x8    ] = ff_pred8x8_vertical_10_sse2;
+                h->pred8x8[HOR_PRED8x8     ] = ff_pred8x8_horizontal_10_sse2;
+            }
 
             h->pred8x8l[VERT_PRED           ] = ff_pred8x8l_vertical_10_sse2;
             h->pred8x8l[HOR_PRED            ] = ff_pred8x8l_horizontal_10_sse2;
diff --git a/libavcodec/x86/h264dsp_mmx.c b/libavcodec/x86/h264dsp_mmx.c
index 68e543681f..71beb262c9 100644
--- a/libavcodec/x86/h264dsp_mmx.c
+++ b/libavcodec/x86/h264dsp_mmx.c
@@ -350,7 +350,7 @@ H264_BIWEIGHT_10_SSE( 4,  8, 10)
 H264_BIWEIGHT_10_SSE( 4,  4, 10)
 H264_BIWEIGHT_10_SSE( 4,  2, 10)
 
-void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth)
+void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth, const int chroma_format_idc)
 {
     int mm_flags = av_get_cpu_flags();
 
@@ -368,7 +368,8 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth)
 
         c->h264_idct_add16          = ff_h264_idct_add16_8_mmx;
         c->h264_idct8_add4          = ff_h264_idct8_add4_8_mmx;
-        c->h264_idct_add8           = ff_h264_idct_add8_8_mmx;
+        if (chroma_format_idc == 1)
+            c->h264_idct_add8       = ff_h264_idct_add8_8_mmx;
         c->h264_idct_add16intra     = ff_h264_idct_add16intra_8_mmx;
         c->h264_luma_dc_dequant_idct= ff_h264_luma_dc_dequant_idct_mmx;
 
@@ -377,13 +378,16 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth)
             c->h264_idct8_dc_add   = ff_h264_idct8_dc_add_8_mmx2;
             c->h264_idct_add16     = ff_h264_idct_add16_8_mmx2;
             c->h264_idct8_add4     = ff_h264_idct8_add4_8_mmx2;
-            c->h264_idct_add8      = ff_h264_idct_add8_8_mmx2;
+            if (chroma_format_idc == 1)
+                c->h264_idct_add8  = ff_h264_idct_add8_8_mmx2;
             c->h264_idct_add16intra= ff_h264_idct_add16intra_8_mmx2;
 
             c->h264_v_loop_filter_chroma= ff_deblock_v_chroma_8_mmxext;
-            c->h264_h_loop_filter_chroma= ff_deblock_h_chroma_8_mmxext;
             c->h264_v_loop_filter_chroma_intra= ff_deblock_v_chroma_intra_8_mmxext;
-            c->h264_h_loop_filter_chroma_intra= ff_deblock_h_chroma_intra_8_mmxext;
+            if (chroma_format_idc == 1) {
+                c->h264_h_loop_filter_chroma= ff_deblock_h_chroma_8_mmxext;
+                c->h264_h_loop_filter_chroma_intra= ff_deblock_h_chroma_intra_8_mmxext;
+            }
 #if ARCH_X86_32
             c->h264_v_loop_filter_luma= ff_deblock_v_luma_8_mmxext;
             c->h264_h_loop_filter_luma= ff_deblock_h_luma_8_mmxext;
@@ -413,7 +417,8 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth)
 
                 c->h264_idct_add16          = ff_h264_idct_add16_8_sse2;
                 c->h264_idct8_add4          = ff_h264_idct8_add4_8_sse2;
-                c->h264_idct_add8           = ff_h264_idct_add8_8_sse2;
+                if (chroma_format_idc == 1)
+                    c->h264_idct_add8       = ff_h264_idct_add8_8_sse2;
                 c->h264_idct_add16intra     = ff_h264_idct_add16intra_8_sse2;
                 c->h264_luma_dc_dequant_idct= ff_h264_luma_dc_dequant_idct_sse2;
 
@@ -472,7 +477,8 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth)
                 c->h264_idct8_dc_add   = ff_h264_idct8_dc_add_10_sse2;
 
                 c->h264_idct_add16     = ff_h264_idct_add16_10_sse2;
-                c->h264_idct_add8      = ff_h264_idct_add8_10_sse2;
+                if (chroma_format_idc == 1)
+                    c->h264_idct_add8  = ff_h264_idct_add8_10_sse2;
                 c->h264_idct_add16intra= ff_h264_idct_add16intra_10_sse2;
 #if HAVE_ALIGNED_STACK
                 c->h264_idct8_add      = ff_h264_idct8_add_10_sse2;
@@ -532,7 +538,8 @@ void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth)
                 c->h264_idct8_dc_add   = ff_h264_idct8_dc_add_10_avx;
 
                 c->h264_idct_add16     = ff_h264_idct_add16_10_avx;
-                c->h264_idct_add8      = ff_h264_idct_add8_10_avx;
+                if (chroma_format_idc == 1)
+                    c->h264_idct_add8  = ff_h264_idct_add8_10_avx;
                 c->h264_idct_add16intra= ff_h264_idct_add16intra_10_avx;
 #if HAVE_ALIGNED_STACK
                 c->h264_idct8_add      = ff_h264_idct8_add_10_avx;