diff options
| author | Michael Niedermayer <michaelni@gmx.at> | 2013-04-16 11:15:47 +0200 |
|---|---|---|
| committer | Michael Niedermayer <michaelni@gmx.at> | 2013-04-16 11:33:40 +0200 |
| commit | bcf643486f468d058cccb3ebc7bfbf8839ad7680 (patch) | |
| tree | 9acc0841ae92887f59e38beec17507aa5d69370e /libswscale/utils.c | |
| parent | 2fa11a94a312aa072d7d14204a09582950bebccb (diff) | |
sws/fill_rgb2yuv_table: increase precission to 32bit
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
Diffstat (limited to 'libswscale/utils.c')
| -rw-r--r-- | libswscale/utils.c | 30 |
1 files changed, 17 insertions, 13 deletions
diff --git a/libswscale/utils.c b/libswscale/utils.c index 71e85e010f..0ea05dd902 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -801,7 +801,7 @@ static void getSubSampleFactors(int *h, int *v, enum AVPixelFormat format) static void fill_rgb2yuv_table(SwsContext *c, const int table[4], int dstRange) { - int64_t W,V,Z; + int64_t W, V, Z, Cy, Cu, Cv; int64_t vr = table[0]; int64_t ub = table[1]; int64_t ug = -table[2]; @@ -858,21 +858,25 @@ static void fill_rgb2yuv_table(SwsContext *c, const int table[4], int dstRange) ug = ug * 224 / 255; vg = vg * 224 / 255; } - W = ONE*ug/ub; - V = ONE*vg/vr; - Z = ONE-W-V; + W = ONE*ONE*ug/ub; + V = ONE*ONE*vg/vr; + Z = ONE*ONE-W-V; - c->input_rgb2yuv_table[RY_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*V/(Z*cy); - c->input_rgb2yuv_table[GY_IDX] = (1 << RGB2YUV_SHIFT)*ONE*ONE/(Z*cy); - c->input_rgb2yuv_table[BY_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*W/(Z*cy); + Cy = cy*Z / ONE; + Cu = ub*Z / ONE; + Cv = vr*Z / ONE; - c->input_rgb2yuv_table[RU_IDX] = (1 << RGB2YUV_SHIFT)*ONE*V/(Z*ub); - c->input_rgb2yuv_table[GU_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*ONE/(Z*ub); - c->input_rgb2yuv_table[BU_IDX] = (1 << RGB2YUV_SHIFT)*ONE*(Z+W)/(Z*ub); + c->input_rgb2yuv_table[RY_IDX] = -(1 << RGB2YUV_SHIFT)*V/Cy; + c->input_rgb2yuv_table[GY_IDX] = (1 << RGB2YUV_SHIFT)*ONE*ONE/Cy; + c->input_rgb2yuv_table[BY_IDX] = -(1 << RGB2YUV_SHIFT)*W/Cy; - c->input_rgb2yuv_table[RV_IDX] = (1 << RGB2YUV_SHIFT)*ONE*(V+Z)/(Z*vr); - c->input_rgb2yuv_table[GV_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*ONE/(Z*vr); - c->input_rgb2yuv_table[BV_IDX] = (1 << RGB2YUV_SHIFT)*ONE*W/(Z*vr); + c->input_rgb2yuv_table[RU_IDX] = (1 << RGB2YUV_SHIFT)*V/Cu; + c->input_rgb2yuv_table[GU_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*ONE/Cu; + c->input_rgb2yuv_table[BU_IDX] = (1 << RGB2YUV_SHIFT)*(Z+W)/Cu; + + c->input_rgb2yuv_table[RV_IDX] = (1 << RGB2YUV_SHIFT)*(V+Z)/Cv; + c->input_rgb2yuv_table[GV_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*ONE/Cv; + c->input_rgb2yuv_table[BV_IDX] = (1 << RGB2YUV_SHIFT)*W/Cv; if(/*!dstRange && */table == ff_yuv2rgb_coeffs[SWS_CS_DEFAULT]) { c->input_rgb2yuv_table[BY_IDX] = ((int)(0.114 * 219 / 255 * (1 << RGB2YUV_SHIFT) + 0.5)); |