diff options
| author | Michael Niedermayer <michael@niedermayer.cc> | 2022-11-02 20:00:38 +0100 |
|---|---|---|
| committer | Michael Niedermayer <michael@niedermayer.cc> | 2022-11-04 22:44:16 +0100 |
| commit | b74f89caaef2174b0bfb2791ea88e44960dba11f (patch) | |
| tree | 8497c8b3dc534cd4928e501c9ad5fdd9882bbf19 /libswscale/x86/output.asm | |
| parent | 0f0afc7fb5d30c40108d81b320823d8f5c9fbedc (diff) | |
swscale/output: Bias 16bps output calculations to improve non overflowing range for GBRP16/GBRPF32
Fixes: integer overflow
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
Diffstat (limited to 'libswscale/x86/output.asm')
| -rw-r--r-- | libswscale/x86/output.asm | 16 |
1 files changed, 15 insertions, 1 deletions
diff --git a/libswscale/x86/output.asm b/libswscale/x86/output.asm index 84e94baaf6..f943a27534 100644 --- a/libswscale/x86/output.asm +++ b/libswscale/x86/output.asm @@ -44,11 +44,13 @@ pd_yuv2gbrp_y_start: times 8 dd (1 << 9) pd_yuv2gbrp_uv_start: times 8 dd ((1 << 9) - (128 << 19)) pd_yuv2gbrp_a_start: times 8 dd (1 << 18) pd_yuv2gbrp16_offset: times 8 dd 0x10000 ;(1 << 16) -pd_yuv2gbrp16_round13: times 8 dd 0x02000 ;(1 << 13) +pd_yuv2gbrp16_round13: times 8 dd 0xE0002000 ;(1 << 13) - (1 << 29) pd_yuv2gbrp16_a_offset: times 8 dd 0x20002000 pd_yuv2gbrp16_upper30: times 8 dd 0x3FFFFFFF ;(1<<30) - 1 pd_yuv2gbrp16_upper27: times 8 dd 0x07FFFFFF ;(1<<27) - 1 +pd_yuv2gbrp16_upper16: times 8 dd 0x0000FFFF ;(1<<16) - 1 pd_yuv2gbrp16_upperC: times 8 dd 0xC0000000 +pd_yuv2gbrp_debias: times 8 dd 0x00008000 ;(1 << 29 - 14) pb_pack_shuffle8: db 0, 4, 8, 12, \ -1, -1, -1, -1, \ -1, -1, -1, -1, \ @@ -883,14 +885,26 @@ cglobal yuv2%1_full_X, 12, 14, 16, ptr, lumFilter, lumSrcx, lumFilterSize, chrFi paddd G, Y paddd B, Y +%if DEPTH < 16 CLIPP2 R, 30 CLIPP2 G, 30 CLIPP2 B, 30 +%endif psrad R, RGB_SHIFT psrad G, RGB_SHIFT psrad B, RGB_SHIFT +%if DEPTH >= 16 + paddd R, [pd_yuv2gbrp_debias] + paddd G, [pd_yuv2gbrp_debias] + paddd B, [pd_yuv2gbrp_debias] + + CLIPP2 R, 16 + CLIPP2 G, 16 + CLIPP2 B, 16 +%endif + %if FLOAT cvtdq2ps R, R cvtdq2ps G, G |