h264: do not use 422 functions for monochrome

Fixes invalid memory access.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC:libav-stable@libav.org

2 files changed, 17 insertions, 17 deletions

diff --git a/libavcodec/x86/h264_intrapred_init.c b/libavcodec/x86/h264_intrapred_init.c
index 6dd98aab72..f934256706 100644
--- a/libavcodec/x86/h264_intrapred_init.c
+++ b/libavcodec/x86/h264_intrapred_init.c
@@ -191,7 +191,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
         if (EXTERNAL_MMX(cpu_flags)) {
             h->pred16x16[VERT_PRED8x8         ] = ff_pred16x16_vertical_8_mmx;
             h->pred16x16[HOR_PRED8x8          ] = ff_pred16x16_horizontal_8_mmx;
-            if (chroma_format_idc == 1) {
+            if (chroma_format_idc <= 1) {
                 h->pred8x8  [VERT_PRED8x8     ] = ff_pred8x8_vertical_8_mmx;
                 h->pred8x8  [HOR_PRED8x8      ] = ff_pred8x8_horizontal_8_mmx;
             }
@@ -200,7 +200,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_8_mmx;
                 h->pred4x4  [TM_VP8_PRED      ] = ff_pred4x4_tm_vp8_8_mmx;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmx;
                 if (codec_id == AV_CODEC_ID_SVQ3) {
                     if (cpu_flags & AV_CPU_FLAG_CMOV)
@@ -216,7 +216,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
         if (EXTERNAL_MMXEXT(cpu_flags)) {
             h->pred16x16[HOR_PRED8x8            ] = ff_pred16x16_horizontal_8_mmxext;
             h->pred16x16[DC_PRED8x8             ] = ff_pred16x16_dc_8_mmxext;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 h->pred8x8[HOR_PRED8x8          ] = ff_pred8x8_horizontal_8_mmxext;
             h->pred8x8l [TOP_DC_PRED            ] = ff_pred8x8l_top_dc_8_mmxext;
             h->pred8x8l [DC_PRED                ] = ff_pred8x8l_dc_8_mmxext;
@@ -241,7 +241,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
                 h->pred4x4  [HOR_UP_PRED        ] = ff_pred4x4_horizontal_up_8_mmxext;
             }
             if (codec_id == AV_CODEC_ID_SVQ3 || codec_id == AV_CODEC_ID_H264) {
-                if (chroma_format_idc == 1) {
+                if (chroma_format_idc <= 1) {
                     h->pred8x8[TOP_DC_PRED8x8   ] = ff_pred8x8_top_dc_8_mmxext;
                     h->pred8x8[DC_PRED8x8       ] = ff_pred8x8_dc_8_mmxext;
                 }
@@ -253,7 +253,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
                 h->pred4x4  [TM_VP8_PRED        ] = ff_pred4x4_tm_vp8_8_mmxext;
                 h->pred4x4  [VERT_PRED          ] = ff_pred4x4_vertical_vp8_8_mmxext;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmxext;
                 if (codec_id == AV_CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8  ] = ff_pred16x16_plane_svq3_8_mmxext;
@@ -280,7 +280,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
                 h->pred16x16[PLANE_PRED8x8    ] = ff_pred16x16_tm_vp8_8_sse2;
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_8_sse2;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_8_sse2;
                 if (codec_id == AV_CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_sse2;
@@ -295,7 +295,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
         if (EXTERNAL_SSSE3(cpu_flags)) {
             h->pred16x16[HOR_PRED8x8          ] = ff_pred16x16_horizontal_8_ssse3;
             h->pred16x16[DC_PRED8x8           ] = ff_pred16x16_dc_8_ssse3;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 h->pred8x8  [HOR_PRED8x8      ] = ff_pred8x8_horizontal_8_ssse3;
             h->pred8x8l [TOP_DC_PRED          ] = ff_pred8x8l_top_dc_8_ssse3;
             h->pred8x8l [DC_PRED              ] = ff_pred8x8l_dc_8_ssse3;
@@ -311,7 +311,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
                 h->pred8x8  [PLANE_PRED8x8    ] = ff_pred8x8_tm_vp8_8_ssse3;
                 h->pred4x4  [TM_VP8_PRED      ] = ff_pred4x4_tm_vp8_8_ssse3;
             } else {
-                if (chroma_format_idc == 1)
+                if (chroma_format_idc <= 1)
                     h->pred8x8  [PLANE_PRED8x8] = ff_pred8x8_plane_8_ssse3;
                 if (codec_id == AV_CODEC_ID_SVQ3) {
                     h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_ssse3;
@@ -327,7 +327,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
             h->pred4x4[DC_PRED             ] = ff_pred4x4_dc_10_mmxext;
             h->pred4x4[HOR_UP_PRED         ] = ff_pred4x4_horizontal_up_10_mmxext;
 
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 h->pred8x8[DC_PRED8x8      ] = ff_pred8x8_dc_10_mmxext;
 
             h->pred8x8l[DC_128_PRED        ] = ff_pred8x8l_128_dc_10_mmxext;
@@ -346,7 +346,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id,
             h->pred4x4[VERT_RIGHT_PRED     ] = ff_pred4x4_vertical_right_10_sse2;
             h->pred4x4[HOR_DOWN_PRED       ] = ff_pred4x4_horizontal_down_10_sse2;
 
-            if (chroma_format_idc == 1) {
+            if (chroma_format_idc <= 1) {
                 h->pred8x8[DC_PRED8x8      ] = ff_pred8x8_dc_10_sse2;
                 h->pred8x8[TOP_DC_PRED8x8  ] = ff_pred8x8_top_dc_10_sse2;
                 h->pred8x8[PLANE_PRED8x8   ] = ff_pred8x8_plane_10_sse2;
diff --git a/libavcodec/x86/h264dsp_init.c b/libavcodec/x86/h264dsp_init.c
index 4164b833db..e9d93e0af9 100644
--- a/libavcodec/x86/h264dsp_init.c
+++ b/libavcodec/x86/h264dsp_init.c
@@ -213,7 +213,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
 {
     int cpu_flags = av_get_cpu_flags();
 
-    if (chroma_format_idc == 1 && EXTERNAL_MMXEXT(cpu_flags))
+    if (chroma_format_idc <= 1 && EXTERNAL_MMXEXT(cpu_flags))
         c->h264_loop_filter_strength = ff_h264_loop_filter_strength_mmxext;
 
     if (bit_depth == 8) {
@@ -225,7 +225,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
 
             c->h264_idct_add16 = ff_h264_idct_add16_8_mmx;
             c->h264_idct8_add4 = ff_h264_idct8_add4_8_mmx;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 c->h264_idct_add8 = ff_h264_idct_add8_8_mmx;
             c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmx;
             if (cpu_flags & AV_CPU_FLAG_CMOV)
@@ -236,13 +236,13 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
             c->h264_idct8_dc_add = ff_h264_idct8_dc_add_8_mmxext;
             c->h264_idct_add16   = ff_h264_idct_add16_8_mmxext;
             c->h264_idct8_add4   = ff_h264_idct8_add4_8_mmxext;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 c->h264_idct_add8 = ff_h264_idct_add8_8_mmxext;
             c->h264_idct_add16intra = ff_h264_idct_add16intra_8_mmxext;
 
             c->h264_v_loop_filter_chroma       = ff_deblock_v_chroma_8_mmxext;
             c->h264_v_loop_filter_chroma_intra = ff_deblock_v_chroma_intra_8_mmxext;
-            if (chroma_format_idc == 1) {
+            if (chroma_format_idc <= 1) {
                 c->h264_h_loop_filter_chroma       = ff_deblock_h_chroma_8_mmxext;
                 c->h264_h_loop_filter_chroma_intra = ff_deblock_h_chroma_intra_8_mmxext;
             }
@@ -265,7 +265,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
 
             c->h264_idct_add16 = ff_h264_idct_add16_8_sse2;
             c->h264_idct8_add4 = ff_h264_idct8_add4_8_sse2;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 c->h264_idct_add8 = ff_h264_idct_add8_8_sse2;
             c->h264_idct_add16intra      = ff_h264_idct_add16intra_8_sse2;
             c->h264_luma_dc_dequant_idct = ff_h264_luma_dc_dequant_idct_sse2;
@@ -308,7 +308,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
             c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_sse2;
 
             c->h264_idct_add16 = ff_h264_idct_add16_10_sse2;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 c->h264_idct_add8 = ff_h264_idct_add8_10_sse2;
             c->h264_idct_add16intra = ff_h264_idct_add16intra_10_sse2;
 #if HAVE_ALIGNED_STACK
@@ -348,7 +348,7 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
             c->h264_idct8_dc_add = ff_h264_idct8_dc_add_10_avx;
 
             c->h264_idct_add16 = ff_h264_idct_add16_10_avx;
-            if (chroma_format_idc == 1)
+            if (chroma_format_idc <= 1)
                 c->h264_idct_add8 = ff_h264_idct_add8_10_avx;
             c->h264_idct_add16intra = ff_h264_idct_add16intra_10_avx;
 #if HAVE_ALIGNED_STACK