From a03a642d5ceb5f2f7c6ebbf56ff365dfbcdb65eb Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Thu, 28 Nov 2013 10:54:35 +0100 Subject: h264: do not use 422 functions for monochrome Fixes invalid memory access. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC:libav-stable@libav.org --- libavcodec/x86/h264_intrapred_init.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'libavcodec/x86/h264_intrapred_init.c') diff --git a/libavcodec/x86/h264_intrapred_init.c b/libavcodec/x86/h264_intrapred_init.c index 6dd98aab72..f934256706 100644 --- a/libavcodec/x86/h264_intrapred_init.c +++ b/libavcodec/x86/h264_intrapred_init.c @@ -191,7 +191,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, if (EXTERNAL_MMX(cpu_flags)) { h->pred16x16[VERT_PRED8x8 ] = ff_pred16x16_vertical_8_mmx; h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_horizontal_8_mmx; - if (chroma_format_idc == 1) { + if (chroma_format_idc <= 1) { h->pred8x8 [VERT_PRED8x8 ] = ff_pred8x8_vertical_8_mmx; h->pred8x8 [HOR_PRED8x8 ] = ff_pred8x8_horizontal_8_mmx; } @@ -200,7 +200,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred8x8 [PLANE_PRED8x8 ] = ff_pred8x8_tm_vp8_8_mmx; h->pred4x4 [TM_VP8_PRED ] = ff_pred4x4_tm_vp8_8_mmx; } else { - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmx; if (codec_id == AV_CODEC_ID_SVQ3) { if (cpu_flags & AV_CPU_FLAG_CMOV) @@ -216,7 +216,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, if (EXTERNAL_MMXEXT(cpu_flags)) { h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_horizontal_8_mmxext; h->pred16x16[DC_PRED8x8 ] = ff_pred16x16_dc_8_mmxext; - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8[HOR_PRED8x8 ] = ff_pred8x8_horizontal_8_mmxext; h->pred8x8l [TOP_DC_PRED ] = ff_pred8x8l_top_dc_8_mmxext; h->pred8x8l [DC_PRED ] = ff_pred8x8l_dc_8_mmxext; @@ -241,7 +241,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred4x4 [HOR_UP_PRED ] = ff_pred4x4_horizontal_up_8_mmxext; } if (codec_id == AV_CODEC_ID_SVQ3 || codec_id == AV_CODEC_ID_H264) { - if (chroma_format_idc == 1) { + if (chroma_format_idc <= 1) { h->pred8x8[TOP_DC_PRED8x8 ] = ff_pred8x8_top_dc_8_mmxext; h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_8_mmxext; } @@ -253,7 +253,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred4x4 [TM_VP8_PRED ] = ff_pred4x4_tm_vp8_8_mmxext; h->pred4x4 [VERT_PRED ] = ff_pred4x4_vertical_vp8_8_mmxext; } else { - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_mmxext; if (codec_id == AV_CODEC_ID_SVQ3) { h->pred16x16[PLANE_PRED8x8 ] = ff_pred16x16_plane_svq3_8_mmxext; @@ -280,7 +280,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred16x16[PLANE_PRED8x8 ] = ff_pred16x16_tm_vp8_8_sse2; h->pred8x8 [PLANE_PRED8x8 ] = ff_pred8x8_tm_vp8_8_sse2; } else { - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_sse2; if (codec_id == AV_CODEC_ID_SVQ3) { h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_sse2; @@ -295,7 +295,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, if (EXTERNAL_SSSE3(cpu_flags)) { h->pred16x16[HOR_PRED8x8 ] = ff_pred16x16_horizontal_8_ssse3; h->pred16x16[DC_PRED8x8 ] = ff_pred16x16_dc_8_ssse3; - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8 [HOR_PRED8x8 ] = ff_pred8x8_horizontal_8_ssse3; h->pred8x8l [TOP_DC_PRED ] = ff_pred8x8l_top_dc_8_ssse3; h->pred8x8l [DC_PRED ] = ff_pred8x8l_dc_8_ssse3; @@ -311,7 +311,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred8x8 [PLANE_PRED8x8 ] = ff_pred8x8_tm_vp8_8_ssse3; h->pred4x4 [TM_VP8_PRED ] = ff_pred4x4_tm_vp8_8_ssse3; } else { - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8 [PLANE_PRED8x8] = ff_pred8x8_plane_8_ssse3; if (codec_id == AV_CODEC_ID_SVQ3) { h->pred16x16[PLANE_PRED8x8] = ff_pred16x16_plane_svq3_8_ssse3; @@ -327,7 +327,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred4x4[DC_PRED ] = ff_pred4x4_dc_10_mmxext; h->pred4x4[HOR_UP_PRED ] = ff_pred4x4_horizontal_up_10_mmxext; - if (chroma_format_idc == 1) + if (chroma_format_idc <= 1) h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_10_mmxext; h->pred8x8l[DC_128_PRED ] = ff_pred8x8l_128_dc_10_mmxext; @@ -346,7 +346,7 @@ av_cold void ff_h264_pred_init_x86(H264PredContext *h, int codec_id, h->pred4x4[VERT_RIGHT_PRED ] = ff_pred4x4_vertical_right_10_sse2; h->pred4x4[HOR_DOWN_PRED ] = ff_pred4x4_horizontal_down_10_sse2; - if (chroma_format_idc == 1) { + if (chroma_format_idc <= 1) { h->pred8x8[DC_PRED8x8 ] = ff_pred8x8_dc_10_sse2; h->pred8x8[TOP_DC_PRED8x8 ] = ff_pred8x8_top_dc_10_sse2; h->pred8x8[PLANE_PRED8x8 ] = ff_pred8x8_plane_10_sse2; -- cgit v1.2.3