From a03a642d5ceb5f2f7c6ebbf56ff365dfbcdb65eb Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Thu, 28 Nov 2013 10:54:35 +0100 Subject: h264: do not use 422 functions for monochrome Fixes invalid memory access. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC:libav-stable@libav.org --- libavcodec/h264pred.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'libavcodec/h264pred.c') diff --git a/libavcodec/h264pred.c b/libavcodec/h264pred.c index 2c991fd414..0d47897dff 100644 --- a/libavcodec/h264pred.c +++ b/libavcodec/h264pred.c @@ -472,7 +472,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id, h->pred8x8l[TOP_DC_PRED ]= FUNCC(pred8x8l_top_dc , depth);\ h->pred8x8l[DC_128_PRED ]= FUNCC(pred8x8l_128_dc , depth);\ \ - if (chroma_format_idc == 1) {\ + if (chroma_format_idc <= 1) {\ h->pred8x8[VERT_PRED8x8 ]= FUNCC(pred8x8_vertical , depth);\ h->pred8x8[HOR_PRED8x8 ]= FUNCC(pred8x8_horizontal , depth);\ } else {\ @@ -480,7 +480,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id, h->pred8x8[HOR_PRED8x8 ]= FUNCC(pred8x16_horizontal , depth);\ }\ if (codec_id != AV_CODEC_ID_VP8) {\ - if (chroma_format_idc == 1) {\ + if (chroma_format_idc <= 1) {\ h->pred8x8[PLANE_PRED8x8]= FUNCC(pred8x8_plane , depth);\ } else {\ h->pred8x8[PLANE_PRED8x8]= FUNCC(pred8x16_plane , depth);\ @@ -488,7 +488,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id, } else\ h->pred8x8[PLANE_PRED8x8]= FUNCD(pred8x8_tm_vp8);\ if(codec_id != AV_CODEC_ID_RV40 && codec_id != AV_CODEC_ID_VP8){\ - if (chroma_format_idc == 1) {\ + if (chroma_format_idc <= 1) {\ h->pred8x8[DC_PRED8x8 ]= FUNCC(pred8x8_dc , depth);\ h->pred8x8[LEFT_DC_PRED8x8]= FUNCC(pred8x8_left_dc , depth);\ h->pred8x8[TOP_DC_PRED8x8 ]= FUNCC(pred8x8_top_dc , depth);\ @@ -514,7 +514,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id, h->pred8x8[DC_129_PRED8x8]= FUNCC(pred8x8_129_dc , depth);\ }\ }\ - if (chroma_format_idc == 1) {\ + if (chroma_format_idc <= 1) {\ h->pred8x8[DC_128_PRED8x8 ]= FUNCC(pred8x8_128_dc , depth);\ } else {\ h->pred8x8[DC_128_PRED8x8 ]= FUNCC(pred8x16_128_dc , depth);\ @@ -548,7 +548,7 @@ av_cold void ff_h264_pred_init(H264PredContext *h, int codec_id, h->pred4x4_add [ HOR_PRED ]= FUNCC(pred4x4_horizontal_add , depth);\ h->pred8x8l_add [VERT_PRED ]= FUNCC(pred8x8l_vertical_add , depth);\ h->pred8x8l_add [ HOR_PRED ]= FUNCC(pred8x8l_horizontal_add , depth);\ - if (chroma_format_idc == 1) {\ + if (chroma_format_idc <= 1) {\ h->pred8x8_add [VERT_PRED8x8]= FUNCC(pred8x8_vertical_add , depth);\ h->pred8x8_add [ HOR_PRED8x8]= FUNCC(pred8x8_horizontal_add , depth);\ } else {\ -- cgit v1.2.3