diff options
| author | Vittorio Giovara <vittorio.giovara@gmail.com> | 2014-11-04 09:37:04 -0500 |
|---|---|---|
| committer | Vittorio Giovara <vittorio.giovara@gmail.com> | 2014-11-06 10:44:46 -0500 |
| commit | 0a6664706168dc1049967bec311970d720581625 (patch) | |
| tree | 05e736195d9652db7e8f17b630e4149216d7eaf9 /libavcodec/mpegvideo_enc.c | |
| parent | e0a1d0a2b04eb5220d00fc7ce46a57cc5e3c7118 (diff) | |
mpegvideo_enc: factor out denominator and explicitly cast operands
CC: libav-stable@libav.org
Bug-Id: CID 608053
Diffstat (limited to 'libavcodec/mpegvideo_enc.c')
| -rw-r--r-- | libavcodec/mpegvideo_enc.c | 16 |
1 files changed, 7 insertions, 9 deletions
diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c index 9006c653c9..8775eac1ce 100644 --- a/libavcodec/mpegvideo_enc.c +++ b/libavcodec/mpegvideo_enc.c @@ -95,42 +95,40 @@ void ff_convert_matrix(MpegEncContext *s, int (*qmat)[64], fdsp->fdct == ff_jpeg_fdct_islow_10) { for (i = 0; i < 64; i++) { const int j = s->idsp.idct_permutation[i]; + int64_t den = (int64_t) qscale * quant_matrix[j]; /* 16 <= qscale * quant_matrix[i] <= 7905 * Assume x = ff_aanscales[i] * qscale * quant_matrix[i] * 19952 <= x <= 249205026 * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026 * 3444240 >= (1 << 36) / (x) >= 275 */ - qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / - (qscale * quant_matrix[j])); + qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den); } } else if (fdsp->fdct == ff_fdct_ifast) { for (i = 0; i < 64; i++) { const int j = s->idsp.idct_permutation[i]; + int64_t den = ff_aanscales[i] * (int64_t) qscale * quant_matrix[j]; /* 16 <= qscale * quant_matrix[i] <= 7905 * Assume x = ff_aanscales[i] * qscale * quant_matrix[i] * 19952 <= x <= 249205026 * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026 * 3444240 >= (1 << 36) / (x) >= 275 */ - qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) / - (ff_aanscales[i] * qscale * - quant_matrix[j])); + qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) / den); } } else { for (i = 0; i < 64; i++) { const int j = s->idsp.idct_permutation[i]; + int64_t den = (int64_t) qscale * quant_matrix[j]; /* We can safely suppose that 16 <= quant_matrix[i] <= 255 * Assume x = qscale * quant_matrix[i] * So 16 <= x <= 7905 * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905 * so 32768 >= (1 << 19) / (x) >= 67 */ - qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / - (qscale * quant_matrix[j])); + qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den); //qmat [qscale][i] = (1 << QMAT_SHIFT_MMX) / // (qscale * quant_matrix[i]); - qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / - (qscale * quant_matrix[j]); + qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / den; if (qmat16[qscale][0][i] == 0 || qmat16[qscale][0][i] == 128 * 256) |