diff options
| author | Michael Niedermayer <michaelni@gmx.at> | 2013-07-13 02:24:56 +0200 |
|---|---|---|
| committer | Michael Niedermayer <michaelni@gmx.at> | 2013-07-13 12:44:37 +0200 |
| commit | f57119b8e58cb5437c3ab40d797293ecb9b4a894 (patch) | |
| tree | 1c621704c0e7755bc05170655841595012f978f1 /libavcodec/j2kenc.c | |
| parent | dc072c98677fde8d716e20e70b147989aac232e1 (diff) | |
jpeg2000: fix overflow in dequantization
Fixes decoding of file generated with:
ffmpeg -f lavfi -i smptehdbars=hd720 -pix_fmt rgb48 /tmp/o.jp2
Reviewed-by: Nicolas BERTRAND <nicoinattendu@gmail.com>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
Diffstat (limited to 'libavcodec/j2kenc.c')
| -rw-r--r-- | libavcodec/j2kenc.c | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c index 48c1db8071..fa15aad120 100644 --- a/libavcodec/j2kenc.c +++ b/libavcodec/j2kenc.c @@ -802,7 +802,7 @@ static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) Jpeg2000Cblk *cblk = prec->cblk + cblkno; cblk->ninclpasses = getcut(cblk, s->lambda, - (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 16); + (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15); } } } @@ -863,7 +863,7 @@ static int encode_tile(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile, int tileno int *ptr = t1.data[y-yy0]; for (x = xx0; x < xx1; x++){ *ptr = (comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x]); - *ptr = (int64_t)*ptr * (int64_t)(16384 * 65536 / band->i_stepsize) >> 14 - NMSEDEC_FRACBITS; + *ptr = (int64_t)*ptr * (int64_t)(16384 * 65536 / band->i_stepsize) >> 15 - NMSEDEC_FRACBITS; ptr++; } } |