diff options
| author | Guillaume Martres <smarter@ubuntu.com> | 2014-01-11 22:46:24 +0100 |
|---|---|---|
| committer | Michael Niedermayer <michaelni@gmx.at> | 2014-01-12 00:49:11 +0100 |
| commit | c9fe0caf7a1abde7ca0b1a359f551103064867b1 (patch) | |
| tree | 5e2daab15f04d19bd0315ac5696b68bf3f532289 /libavcodec/hevcdsp_template.c | |
| parent | 65801040c648bbe352ad48ced2045bbeb25e9fcf (diff) | |
hevc: clip pixels when transquant bypass is used
Fixes: asan_stack-oob_eae8e3_7333_WPP_B_ericsson_MAIN10_2.bit
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
This is a more proper fix than 5856bca360c5bc3e340a357d91b1f993c80a7bea
The reconstructed picture should always be clipped (see section 8.6.5),
previously we did not clip coding units where
cu_transquant_bypass_flag == 1
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
Diffstat (limited to 'libavcodec/hevcdsp_template.c')
| -rw-r--r-- | libavcodec/hevcdsp_template.c | 8 |
1 files changed, 4 insertions, 4 deletions
diff --git a/libavcodec/hevcdsp_template.c b/libavcodec/hevcdsp_template.c index a876ee4625..c0a62bc4ec 100644 --- a/libavcodec/hevcdsp_template.c +++ b/libavcodec/hevcdsp_template.c @@ -52,7 +52,7 @@ static void FUNC(transquant_bypass4x4)(uint8_t *_dst, int16_t *coeffs, for (y = 0; y < 4; y++) { for (x = 0; x < 4; x++) { - dst[x] += *coeffs; + dst[x] = av_clip_pixel(dst[x] + *coeffs); coeffs++; } dst += stride; @@ -69,7 +69,7 @@ static void FUNC(transquant_bypass8x8)(uint8_t *_dst, int16_t *coeffs, for (y = 0; y < 8; y++) { for (x = 0; x < 8; x++) { - dst[x] += *coeffs; + dst[x] = av_clip_pixel(dst[x] + *coeffs); coeffs++; } dst += stride; @@ -86,7 +86,7 @@ static void FUNC(transquant_bypass16x16)(uint8_t *_dst, int16_t *coeffs, for (y = 0; y < 16; y++) { for (x = 0; x < 16; x++) { - dst[x] += *coeffs; + dst[x] = av_clip_pixel(dst[x] + *coeffs); coeffs++; } dst += stride; @@ -103,7 +103,7 @@ static void FUNC(transquant_bypass32x32)(uint8_t *_dst, int16_t *coeffs, for (y = 0; y < 32; y++) { for (x = 0; x < 32; x++) { - dst[x] += *coeffs; + dst[x] = av_clip_pixel(dst[x] + *coeffs); coeffs++; } dst += stride; |