From 0b7e5d0d75e7d8762dd04d35f8c0821736164372 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Mon, 13 Oct 2014 16:02:42 +0200 Subject: postproc: fix qp count Found-by: ubitux Signed-off-by: Michael Niedermayer --- libpostproc/postprocess.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libpostproc/postprocess.c') diff --git a/libpostproc/postprocess.c b/libpostproc/postprocess.c index e3643ae381..f8fb356391 100644 --- a/libpostproc/postprocess.c +++ b/libpostproc/postprocess.c @@ -994,7 +994,7 @@ void pp_postprocess(const uint8_t * src[3], const int srcStride[3], if(pict_type & PP_PICT_TYPE_QP2){ int i; - const int count= mbHeight * absQPStride; + const int count= FFMAX(mbHeight * absQPStride, mbWidth); for(i=0; i<(count>>2); i++){ ((uint32_t*)c->stdQPTable)[i] = (((const uint32_t*)QP_store)[i]>>1) & 0x7F7F7F7F; } @@ -1019,7 +1019,7 @@ void pp_postprocess(const uint8_t * src[3], const int srcStride[3], if((pict_type&7)!=3){ if (QPStride >= 0){ int i; - const int count= mbHeight * FFMAX(QPStride, mbWidth); + const int count= FFMAX(mbHeight * QPStride, mbWidth); for(i=0; i<(count>>2); i++){ ((uint32_t*)c->nonBQPTable)[i] = ((const uint32_t*)QP_store)[i] & 0x3F3F3F3F; } -- cgit v1.2.3