diff options
| author | Loren Merritt <lorenm@u.washington.edu> | 2012-09-22 01:43:16 +0200 |
|---|---|---|
| committer | Anton Khirnov <anton@khirnov.net> | 2013-03-13 09:14:59 +0100 |
| commit | 5b3c1aecb253828d09fa9825c5a4aed97badf086 (patch) | |
| tree | 0393c942f9e6f4593bb69a3698a6969314fb9cc8 /libavfilter | |
| parent | 0fe4b48540613127f7eed13769e989662cb74eef (diff) | |
hqdn3d: Fix out of array read in LOWPASS
CC:libav-stable@libav.org
Signed-off-by: Anton Khirnov <anton@khirnov.net>
Diffstat (limited to 'libavfilter')
| -rw-r--r-- | libavfilter/vf_hqdn3d.c | 8 | ||||
| -rw-r--r-- | libavfilter/x86/vf_hqdn3d.asm | 2 |
2 files changed, 5 insertions, 5 deletions
diff --git a/libavfilter/vf_hqdn3d.c b/libavfilter/vf_hqdn3d.c index 50e904b940..eb0ea6519a 100644 --- a/libavfilter/vf_hqdn3d.c +++ b/libavfilter/vf_hqdn3d.c @@ -37,10 +37,10 @@ #include "vf_hqdn3d.h" #define LUT_BITS (depth==16 ? 8 : 4) -#define RIGHTSHIFT(a,b) (((a)+(((1<<(b))-1)>>1))>>(b)) -#define LOAD(x) ((depth==8 ? src[x] : AV_RN16A(src+(x)*2)) << (16-depth)) -#define STORE(x,val) (depth==8 ? dst[x] = RIGHTSHIFT(val, 16-depth)\ - : AV_WN16A(dst+(x)*2, RIGHTSHIFT(val, 16-depth))) +#define LOAD(x) (((depth == 8 ? src[x] : AV_RN16A(src + (x) * 2)) << (16 - depth))\ + + (((1 << (16 - depth)) - 1) >> 1)) +#define STORE(x,val) (depth == 8 ? dst[x] = (val) >> (16 - depth) : \ + AV_WN16A(dst + (x) * 2, (val) >> (16 - depth))) av_always_inline static uint32_t lowpass(int prev, int cur, int16_t *coef, int depth) diff --git a/libavfilter/x86/vf_hqdn3d.asm b/libavfilter/x86/vf_hqdn3d.asm index dee2c96131..02632a1f09 100644 --- a/libavfilter/x86/vf_hqdn3d.asm +++ b/libavfilter/x86/vf_hqdn3d.asm @@ -39,6 +39,7 @@ SECTION .text %endif %if %3 != 16 shl %1, 16-%3 + add %1, (1<<(15-%3))-1 %endif %endmacro @@ -86,7 +87,6 @@ ALIGN 16 mov [frameantq+xq*2], t0w movifnidn dstq, dstmp %if %1 != 16 - add t0d, (1<<(15-%1))-1 shr t0d, 16-%1 ; could eliminate this by storing from t0h, but only with some contraints on register allocation %endif %if %1 == 8 |