From 62844c3fd66940c7747e9b2bb7804e265319f43f Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 18 Feb 2013 21:03:02 -0800 Subject: h264: Integrate clear_blocks calls with IDCT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The non-intra-pcm branch in hl_decode_mb (simple, 8bpp) goes from 700 to 672 cycles, and the complete loop of decode_mb_cabac and hl_decode_mb (in the decode_slice loop) goes from 1759 to 1733 cycles on the clip tested (cathedral), i.e. almost 30 cycles per mb faster. Signed-off-by: Martin Storsjö --- libavcodec/arm/h264idct_neon.S | 29 +++++++--- libavcodec/h264.c | 14 ++--- libavcodec/h264_mb_template.c | 11 +--- libavcodec/h264addpx_template.c | 4 ++ libavcodec/h264dsp.c | 4 +- libavcodec/h264dsp.h | 4 +- libavcodec/h264idct_template.c | 16 ++++-- libavcodec/h264pred.h | 8 +-- libavcodec/h264pred_template.c | 28 ++++++---- libavcodec/ppc/h264_altivec.c | 3 ++ libavcodec/svq3.c | 4 +- libavcodec/x86/h264_idct.asm | 108 ++++++++++++++++++++++++++++--------- libavcodec/x86/h264_idct_10bit.asm | 53 ++++++++++++++++-- 13 files changed, 209 insertions(+), 77 deletions(-) diff --git a/libavcodec/arm/h264idct_neon.S b/libavcodec/arm/h264idct_neon.S index b23ddb1b73..3e5321cd05 100644 --- a/libavcodec/arm/h264idct_neon.S +++ b/libavcodec/arm/h264idct_neon.S @@ -22,9 +22,12 @@ function ff_h264_idct_add_neon, export=1 vld1.64 {d0-d3}, [r1,:128] + vmov.i16 q15, #0 vswp d1, d2 + vst1.16 {q15}, [r1,:128]! vadd.i16 d4, d0, d1 + vst1.16 {q15}, [r1,:128]! vshr.s16 q8, q1, #1 vsub.i16 d5, d0, d1 vadd.i16 d6, d2, d17 @@ -65,11 +68,14 @@ function ff_h264_idct_add_neon, export=1 vst1.32 {d0[1]}, [r0,:32], r2 vst1.32 {d1[0]}, [r0,:32], r2 + sub r1, r1, #32 bx lr endfunc function ff_h264_idct_dc_add_neon, export=1 + mov r3, #0 vld1.16 {d2[],d3[]}, [r1,:16] + strh r3, [r1] vrshr.s16 q1, q1, #6 vld1.32 {d0[0]}, [r0,:32], r2 vld1.32 {d0[1]}, [r0,:32], r2 @@ -148,7 +154,7 @@ function ff_h264_idct_add8_neon, export=1 add r5, r1, #16*4 add r1, r2, #16*32 mov r2, r3 - mov r3, r1 + mov r10, r1 ldr r6, [sp, #32] movrel r7, scan8+16 mov r12, #0 @@ -156,7 +162,7 @@ function ff_h264_idct_add8_neon, export=1 ldr r0, [r5, r12, lsl #2] ldrb r8, [r6, r8] add r0, r0, r4 - add r1, r3, r12, lsl #5 + add r1, r10, r12, lsl #5 cmp r8, #0 ldrsh r8, [r1] iteet ne @@ -180,7 +186,9 @@ endfunc qb .req q14 vshr.s16 q2, q10, #1 vadd.i16 q0, q8, q12 - vld1.16 {q14-q15},[r1,:128]! + vld1.16 {q14-q15},[r1,:128] + vst1.16 {q3}, [r1,:128]! + vst1.16 {q3}, [r1,:128]! vsub.i16 q1, q8, q12 vshr.s16 q3, q14, #1 vsub.i16 q2, q2, q14 @@ -259,9 +267,16 @@ endfunc .endm function ff_h264_idct8_add_neon, export=1 - vld1.16 {q8-q9}, [r1,:128]! - vld1.16 {q10-q11},[r1,:128]! - vld1.16 {q12-q13},[r1,:128]! + vmov.i16 q3, #0 + vld1.16 {q8-q9}, [r1,:128] + vst1.16 {q3}, [r1,:128]! + vst1.16 {q3}, [r1,:128]! + vld1.16 {q10-q11},[r1,:128] + vst1.16 {q3}, [r1,:128]! + vst1.16 {q3}, [r1,:128]! + vld1.16 {q12-q13},[r1,:128] + vst1.16 {q3}, [r1,:128]! + vst1.16 {q3}, [r1,:128]! idct8x8_cols 0 idct8x8_cols 1 @@ -313,7 +328,9 @@ function ff_h264_idct8_add_neon, export=1 endfunc function ff_h264_idct8_dc_add_neon, export=1 + mov r3, #0 vld1.16 {d30[],d31[]},[r1,:16] + strh r3, [r1] vld1.32 {d0}, [r0,:64], r2 vrshr.s16 q15, q15, #6 vld1.32 {d1}, [r0,:64], r2 diff --git a/libavcodec/h264.c b/libavcodec/h264.c index bfe9940738..7d7f37601a 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -2257,7 +2257,7 @@ static av_always_inline void hl_decode_mb_predict_luma(H264Context *h, if (IS_8x8DCT(mb_type)) { if (transform_bypass) { idct_dc_add = - idct_add = h->h264dsp.h264_add_pixels8; + idct_add = h->h264dsp.h264_add_pixels8_clear; } else { idct_dc_add = h->h264dsp.h264_idct8_dc_add; idct_add = h->h264dsp.h264_idct8_add; @@ -2282,7 +2282,7 @@ static av_always_inline void hl_decode_mb_predict_luma(H264Context *h, } else { if (transform_bypass) { idct_dc_add = - idct_add = h->h264dsp.h264_add_pixels4; + idct_add = h->h264dsp.h264_add_pixels4_clear; } else { idct_dc_add = h->h264dsp.h264_idct_dc_add; idct_add = h->h264dsp.h264_idct_add; @@ -2379,9 +2379,9 @@ static av_always_inline void hl_decode_mb_idct_luma(H264Context *h, int mb_type, for (i = 0; i < 16; i++) if (h->non_zero_count_cache[scan8[i + p * 16]] || dctcoef_get(h->mb, pixel_shift, i * 16 + p * 256)) - h->h264dsp.h264_add_pixels4(dest_y + block_offset[i], - h->mb + (i * 16 + p * 256 << pixel_shift), - linesize); + h->h264dsp.h264_add_pixels4_clear(dest_y + block_offset[i], + h->mb + (i * 16 + p * 256 << pixel_shift), + linesize); } } else { h->h264dsp.h264_idct_add16intra(dest_y, block_offset, @@ -2392,8 +2392,8 @@ static av_always_inline void hl_decode_mb_idct_luma(H264Context *h, int mb_type, } else if (h->cbp & 15) { if (transform_bypass) { const int di = IS_8x8DCT(mb_type) ? 4 : 1; - idct_add = IS_8x8DCT(mb_type) ? h->h264dsp.h264_add_pixels8 - : h->h264dsp.h264_add_pixels4; + idct_add = IS_8x8DCT(mb_type) ? h->h264dsp.h264_add_pixels8_clear + : h->h264dsp.h264_add_pixels4_clear; for (i = 0; i < 16; i += di) if (h->non_zero_count_cache[scan8[i + p * 16]]) idct_add(dest_y + block_offset[i], diff --git a/libavcodec/h264_mb_template.c b/libavcodec/h264_mb_template.c index 957260a081..9b63fefeda 100644 --- a/libavcodec/h264_mb_template.c +++ b/libavcodec/h264_mb_template.c @@ -207,7 +207,7 @@ static av_noinline void FUNC(hl_decode_mb)(H264Context *h) h->mb + (16 * 16 * 2 << PIXEL_SHIFT), uvlinesize); } else { - idct_add = h->h264dsp.h264_add_pixels4; + idct_add = h->h264dsp.h264_add_pixels4_clear; for (j = 1; j < 3; j++) { for (i = j * 16; i < j * 16 + 4; i++) if (h->non_zero_count_cache[scan8[i]] || @@ -261,10 +261,6 @@ static av_noinline void FUNC(hl_decode_mb)(H264Context *h) } } } - if (h->cbp || IS_INTRA(mb_type)) { - h->dsp.clear_blocks(h->mb); - h->dsp.clear_blocks(h->mb + (24 * 16 << PIXEL_SHIFT)); - } } } @@ -368,11 +364,6 @@ static av_noinline void FUNC(hl_decode_mb_444)(H264Context *h) hl_decode_mb_idct_luma(h, mb_type, 1, SIMPLE, transform_bypass, PIXEL_SHIFT, block_offset, linesize, dest[p], p); - - if (h->cbp || IS_INTRA(mb_type)) { - h->dsp.clear_blocks(h->mb); - h->dsp.clear_blocks(h->mb + (24 * 16 << PIXEL_SHIFT)); - } } } diff --git a/libavcodec/h264addpx_template.c b/libavcodec/h264addpx_template.c index d1babc54e5..cdbfc67dfe 100644 --- a/libavcodec/h264addpx_template.c +++ b/libavcodec/h264addpx_template.c @@ -43,6 +43,8 @@ static void FUNCC(ff_h264_add_pixels4)(uint8_t *_dst, int16_t *_src, int stride) dst += stride; src += 4; } + + memset(_src, 0, sizeof(dctcoef) * 16); } static void FUNCC(ff_h264_add_pixels8)(uint8_t *_dst, int16_t *_src, int stride) @@ -65,4 +67,6 @@ static void FUNCC(ff_h264_add_pixels8)(uint8_t *_dst, int16_t *_src, int stride) dst += stride; src += 8; } + + memset(_src, 0, sizeof(dctcoef) * 64); } diff --git a/libavcodec/h264dsp.c b/libavcodec/h264dsp.c index b739758e24..8a70aefedc 100644 --- a/libavcodec/h264dsp.c +++ b/libavcodec/h264dsp.c @@ -57,8 +57,8 @@ void ff_h264dsp_init(H264DSPContext *c, const int bit_depth, const int chroma_fo #define FUNC(a, depth) a ## _ ## depth ## _c #define ADDPX_DSP(depth) \ - c->h264_add_pixels4 = FUNC(ff_h264_add_pixels4, depth);\ - c->h264_add_pixels8 = FUNC(ff_h264_add_pixels8, depth) + c->h264_add_pixels4_clear = FUNC(ff_h264_add_pixels4, depth);\ + c->h264_add_pixels8_clear = FUNC(ff_h264_add_pixels8, depth) if (bit_depth > 8 && bit_depth <= 16) { ADDPX_DSP(16); diff --git a/libavcodec/h264dsp.h b/libavcodec/h264dsp.h index 757caf8162..1f9f8fe823 100644 --- a/libavcodec/h264dsp.h +++ b/libavcodec/h264dsp.h @@ -103,8 +103,8 @@ typedef struct H264DSPContext { void (*h264_chroma_dc_dequant_idct)(int16_t *block, int qmul); /* bypass-transform */ - void (*h264_add_pixels8)(uint8_t *dst, int16_t *block, int stride); - void (*h264_add_pixels4)(uint8_t *dst, int16_t *block, int stride); + void (*h264_add_pixels8_clear)(uint8_t *dst, int16_t *block, int stride); + void (*h264_add_pixels4_clear)(uint8_t *dst, int16_t *block, int stride); } H264DSPContext; void ff_h264dsp_init(H264DSPContext *c, const int bit_depth, diff --git a/libavcodec/h264idct_template.c b/libavcodec/h264idct_template.c index fc285c57b0..aadafaddd2 100644 --- a/libavcodec/h264idct_template.c +++ b/libavcodec/h264idct_template.c @@ -61,6 +61,8 @@ void FUNCC(ff_h264_idct_add)(uint8_t *_dst, int16_t *_block, int stride) dst[i + 2*stride]= av_clip_pixel(dst[i + 2*stride] + ((z1 - z2) >> 6)); dst[i + 3*stride]= av_clip_pixel(dst[i + 3*stride] + ((z0 - z3) >> 6)); } + + memset(block, 0, 16 * sizeof(dctcoef)); } void FUNCC(ff_h264_idct8_add)(uint8_t *_dst, int16_t *_block, int stride){ @@ -133,14 +135,18 @@ void FUNCC(ff_h264_idct8_add)(uint8_t *_dst, int16_t *_block, int stride){ dst[i + 6*stride] = av_clip_pixel( dst[i + 6*stride] + ((b2 - b5) >> 6) ); dst[i + 7*stride] = av_clip_pixel( dst[i + 7*stride] + ((b0 - b7) >> 6) ); } + + memset(block, 0, 64 * sizeof(dctcoef)); } // assumes all AC coefs are 0 -void FUNCC(ff_h264_idct_dc_add)(uint8_t *_dst, int16_t *block, int stride){ +void FUNCC(ff_h264_idct_dc_add)(uint8_t *_dst, int16_t *_block, int stride){ int i, j; - int dc = (((dctcoef*)block)[0] + 32) >> 6; pixel *dst = (pixel*)_dst; + dctcoef *block = (dctcoef*)_block; + int dc = (block[0] + 32) >> 6; stride /= sizeof(pixel); + block[0] = 0; for( j = 0; j < 4; j++ ) { for( i = 0; i < 4; i++ ) @@ -149,10 +155,12 @@ void FUNCC(ff_h264_idct_dc_add)(uint8_t *_dst, int16_t *block, int stride){ } } -void FUNCC(ff_h264_idct8_dc_add)(uint8_t *_dst, int16_t *block, int stride){ +void FUNCC(ff_h264_idct8_dc_add)(uint8_t *_dst, int16_t *_block, int stride){ int i, j; - int dc = (((dctcoef*)block)[0] + 32) >> 6; pixel *dst = (pixel*)_dst; + dctcoef *block = (dctcoef*)_block; + int dc = (block[0] + 32) >> 6; + block[0] = 0; stride /= sizeof(pixel); for( j = 0; j < 8; j++ ) { diff --git a/libavcodec/h264pred.h b/libavcodec/h264pred.h index ad1739a3c5..62e5b79dc8 100644 --- a/libavcodec/h264pred.h +++ b/libavcodec/h264pred.h @@ -98,15 +98,15 @@ typedef struct H264PredContext { void(*pred16x16[4 + 3 + 2])(uint8_t *src, ptrdiff_t stride); void(*pred4x4_add[2])(uint8_t *pix /*align 4*/, - const int16_t *block /*align 16*/, ptrdiff_t stride); + int16_t *block /*align 16*/, ptrdiff_t stride); void(*pred8x8l_add[2])(uint8_t *pix /*align 8*/, - const int16_t *block /*align 16*/, ptrdiff_t stride); + int16_t *block /*align 16*/, ptrdiff_t stride); void(*pred8x8_add[3])(uint8_t *pix /*align 8*/, const int *block_offset, - const int16_t *block /*align 16*/, ptrdiff_t stride); + int16_t *block /*align 16*/, ptrdiff_t stride); void(*pred16x16_add[3])(uint8_t *pix /*align 16*/, const int *block_offset, - const int16_t *block /*align 16*/, ptrdiff_t stride); + int16_t *block /*align 16*/, ptrdiff_t stride); } H264PredContext; void ff_h264_pred_init(H264PredContext *h, int codec_id, diff --git a/libavcodec/h264pred_template.c b/libavcodec/h264pred_template.c index 8e97d11104..e15c76c849 100644 --- a/libavcodec/h264pred_template.c +++ b/libavcodec/h264pred_template.c @@ -1131,7 +1131,7 @@ static void FUNCC(pred8x8l_horizontal_up)(uint8_t *_src, int has_topleft, #undef PL #undef SRC -static void FUNCC(pred4x4_vertical_add)(uint8_t *_pix, const int16_t *_block, +static void FUNCC(pred4x4_vertical_add)(uint8_t *_pix, int16_t *_block, ptrdiff_t stride) { int i; @@ -1148,9 +1148,11 @@ static void FUNCC(pred4x4_vertical_add)(uint8_t *_pix, const int16_t *_block, pix++; block++; } + + memset(_block, 0, sizeof(dctcoef) * 16); } -static void FUNCC(pred4x4_horizontal_add)(uint8_t *_pix, const int16_t *_block, +static void FUNCC(pred4x4_horizontal_add)(uint8_t *_pix, int16_t *_block, ptrdiff_t stride) { int i; @@ -1166,9 +1168,11 @@ static void FUNCC(pred4x4_horizontal_add)(uint8_t *_pix, const int16_t *_block, pix+= stride; block+= 4; } + + memset(_block, 0, sizeof(dctcoef) * 16); } -static void FUNCC(pred8x8l_vertical_add)(uint8_t *_pix, const int16_t *_block, +static void FUNCC(pred8x8l_vertical_add)(uint8_t *_pix, int16_t *_block, ptrdiff_t stride) { int i; @@ -1189,9 +1193,11 @@ static void FUNCC(pred8x8l_vertical_add)(uint8_t *_pix, const int16_t *_block, pix++; block++; } + + memset(_block, 0, sizeof(dctcoef) * 64); } -static void FUNCC(pred8x8l_horizontal_add)(uint8_t *_pix, const int16_t *_block, +static void FUNCC(pred8x8l_horizontal_add)(uint8_t *_pix, int16_t *_block, ptrdiff_t stride) { int i; @@ -1211,10 +1217,12 @@ static void FUNCC(pred8x8l_horizontal_add)(uint8_t *_pix, const int16_t *_block, pix+= stride; block+= 8; } + + memset(_block, 0, sizeof(dctcoef) * 64); } static void FUNCC(pred16x16_vertical_add)(uint8_t *pix, const int *block_offset, - const int16_t *block, + int16_t *block, ptrdiff_t stride) { int i; @@ -1224,7 +1232,7 @@ static void FUNCC(pred16x16_vertical_add)(uint8_t *pix, const int *block_offset, static void FUNCC(pred16x16_horizontal_add)(uint8_t *pix, const int *block_offset, - const int16_t *block, + int16_t *block, ptrdiff_t stride) { int i; @@ -1233,7 +1241,7 @@ static void FUNCC(pred16x16_horizontal_add)(uint8_t *pix, } static void FUNCC(pred8x8_vertical_add)(uint8_t *pix, const int *block_offset, - const int16_t *block, ptrdiff_t stride) + int16_t *block, ptrdiff_t stride) { int i; for(i=0; i<4; i++) @@ -1241,7 +1249,7 @@ static void FUNCC(pred8x8_vertical_add)(uint8_t *pix, const int *block_offset, } static void FUNCC(pred8x16_vertical_add)(uint8_t *pix, const int *block_offset, - const int16_t *block, ptrdiff_t stride) + int16_t *block, ptrdiff_t stride) { int i; for(i=0; i<4; i++) @@ -1251,7 +1259,7 @@ static void FUNCC(pred8x16_vertical_add)(uint8_t *pix, const int *block_offset, } static void FUNCC(pred8x8_horizontal_add)(uint8_t *pix, const int *block_offset, - const int16_t *block, + int16_t *block, ptrdiff_t stride) { int i; @@ -1261,7 +1269,7 @@ static void FUNCC(pred8x8_horizontal_add)(uint8_t *pix, const int *block_offset, static void FUNCC(pred8x16_horizontal_add)(uint8_t *pix, const int *block_offset, - const int16_t *block, ptrdiff_t stride) + int16_t *block, ptrdiff_t stride) { int i; for(i=0; i<4; i++) diff --git a/libavcodec/ppc/h264_altivec.c b/libavcodec/ppc/h264_altivec.c index 59642fdc36..20549c5b6d 100644 --- a/libavcodec/ppc/h264_altivec.c +++ b/libavcodec/ppc/h264_altivec.c @@ -87,6 +87,7 @@ static void ff_h264_idct_add_altivec(uint8_t *dst, int16_t *block, int stride) vtmp1 = vec_sld(vtmp0, vtmp0, 8); vtmp2 = vec_ld(16,block); vtmp3 = vec_sld(vtmp2, vtmp2, 8); + memset(block, 0, 16 * sizeof(int16_t)); VEC_1D_DCT(vtmp0,vtmp1,vtmp2,vtmp3,va0,va1,va2,va3); VEC_TRANSPOSE_4(va0,va1,va2,va3,vtmp0,vtmp1,vtmp2,vtmp3); @@ -206,6 +207,7 @@ static void ff_h264_idct8_add_altivec( uint8_t *dst, int16_t *dct, int stride ) s5 = vec_ld(0x50, (int16_t*)dct); s6 = vec_ld(0x60, (int16_t*)dct); s7 = vec_ld(0x70, (int16_t*)dct); + memset(dct, 0, 64 * sizeof(int16_t)); IDCT8_1D_ALTIVEC(s0, s1, s2, s3, s4, s5, s6, s7, d0, d1, d2, d3, d4, d5, d6, d7); @@ -234,6 +236,7 @@ static av_always_inline void h264_idct_dc_add_internal(uint8_t *dst, int16_t *bl int i; dc = (block[0] + 32) >> 6; + block[0] = 0; dc16 = vec_splat((vec_s16) vec_lde(0, &dc), 1); if (size == 4) diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c index 1ea99d15a6..0c3af86560 100644 --- a/libavcodec/svq3.c +++ b/libavcodec/svq3.c @@ -216,6 +216,8 @@ void ff_svq3_add_idct_c(uint8_t *dst, int16_t *block, dst[i + stride * 2] = av_clip_uint8(dst[i + stride * 2] + ((z1 - z2) * qmul + rr >> 20)); dst[i + stride * 3] = av_clip_uint8(dst[i + stride * 3] + ((z0 - z3) * qmul + rr >> 20)); } + + memset(block, 0, 16 * sizeof(int16_t)); } static inline int svq3_decode_block(GetBitContext *gb, int16_t *block, @@ -664,8 +666,6 @@ static int svq3_decode_mb(SVQ3Context *s, unsigned int mb_type) } if (!IS_SKIP(mb_type) || h->pict_type == AV_PICTURE_TYPE_B) { memset(h->non_zero_count_cache + 8, 0, 14 * 8 * sizeof(uint8_t)); - h->dsp.clear_blocks(h->mb + 0); - h->dsp.clear_blocks(h->mb + 384); } if (!IS_INTRA16x16(mb_type) && diff --git a/libavcodec/x86/h264_idct.asm b/libavcodec/x86/h264_idct.asm index de0de24428..3ced8af40e 100644 --- a/libavcodec/x86/h264_idct.asm +++ b/libavcodec/x86/h264_idct.asm @@ -70,6 +70,10 @@ SECTION .text paddw m0, m6 IDCT4_1D w, 0, 1, 2, 3, 4, 5 pxor m7, m7 + movq [%2+ 0], m7 + movq [%2+ 8], m7 + movq [%2+16], m7 + movq [%2+24], m7 STORE_DIFFx2 m0, m1, m4, m5, m7, 6, %1, %3 lea %1, [%1+%3*2] @@ -161,13 +165,31 @@ cglobal h264_idct_add_8, 3, 3, 0 %endmacro ; %1=uint8_t *dst, %2=int16_t *block, %3=int stride -%macro IDCT8_ADD_MMX_END 3 +%macro IDCT8_ADD_MMX_END 3-4 IDCT8_1D_FULL %2 mova [%2 ], m5 mova [%2+16], m6 mova [%2+32], m7 pxor m7, m7 +%if %0 == 4 + movq [%4+ 0], m7 + movq [%4+ 8], m7 + movq [%4+ 16], m7 + movq [%4+ 24], m7 + movq [%4+ 32], m7 + movq [%4+ 40], m7 + movq [%4+ 48], m7 + movq [%4+ 56], m7 + movq [%4+ 64], m7 + movq [%4+ 72], m7 + movq [%4+ 80], m7 + movq [%4+ 88], m7 + movq [%4+ 96], m7 + movq [%4+104], m7 + movq [%4+112], m7 + movq [%4+120], m7 +%endif STORE_DIFFx2 m0, m1, m5, m6, m7, 6, %1, %3 lea %1, [%1+%3*2] STORE_DIFFx2 m2, m3, m5, m6, m7, 6, %1, %3 @@ -190,7 +212,7 @@ cglobal h264_idct8_add_8, 3, 4, 0 IDCT8_ADD_MMX_START r1 , rsp IDCT8_ADD_MMX_START r1+8, rsp+64 lea r3, [r0+4] - IDCT8_ADD_MMX_END r0 , rsp, r2 + IDCT8_ADD_MMX_END r0 , rsp, r2, r1 IDCT8_ADD_MMX_END r3 , rsp+8, r2 ADD rsp, pad @@ -233,6 +255,14 @@ cglobal h264_idct8_add_8, 3, 4, 0 SWAP 0, 8 SWAP 1, 9 %endif + mova [%2+ 0], m7 + mova [%2+ 16], m7 + mova [%2+ 32], m7 + mova [%2+ 48], m7 + mova [%2+ 64], m7 + mova [%2+ 80], m7 + mova [%2+ 96], m7 + mova [%2+112], m7 lea %1, [%1+%3*4] STORE_DIFF m4, m6, m7, [%1 ] STORE_DIFF m5, m6, m7, [%1+%3 ] @@ -246,19 +276,11 @@ cglobal h264_idct8_add_8, 3, 4, 10 IDCT8_ADD_SSE r0, r1, r2, r3 RET -%macro DC_ADD_MMXEXT_INIT 2-3 -%if %0 == 2 - movsx %1, word [%1] +%macro DC_ADD_MMXEXT_INIT 2 add %1, 32 sar %1, 6 movd m0, %1d lea %1, [%2*3] -%else - add %3, 32 - sar %3, 6 - movd m0, %3d - lea %3, [%2*3] -%endif pshufw m0, m0, 0 pxor m1, m1 psubw m1, m0 @@ -287,19 +309,44 @@ cglobal h264_idct8_add_8, 3, 4, 10 INIT_MMX mmxext ; ff_h264_idct_dc_add_mmxext(uint8_t *dst, int16_t *block, int stride) -cglobal h264_idct_dc_add_8, 3, 3, 0 - DC_ADD_MMXEXT_INIT r1, r2 - DC_ADD_MMXEXT_OP movh, r0, r2, r1 +%if ARCH_X86_64 +cglobal h264_idct_dc_add_8, 3, 4, 0 + movsx r3, word [r1] + mov dword [r1], 0 + DC_ADD_MMXEXT_INIT r3, r2 + DC_ADD_MMXEXT_OP movh, r0, r2, r3 RET ; ff_h264_idct8_dc_add_mmxext(uint8_t *dst, int16_t *block, int stride) -cglobal h264_idct8_dc_add_8, 3, 3, 0 - DC_ADD_MMXEXT_INIT r1, r2 - DC_ADD_MMXEXT_OP mova, r0, r2, r1 +cglobal h264_idct8_dc_add_8, 3, 4, 0 + movsx r3, word [r1] + mov dword [r1], 0 + DC_ADD_MMXEXT_INIT r3, r2 + DC_ADD_MMXEXT_OP mova, r0, r2, r3 lea r0, [r0+r2*4] - DC_ADD_MMXEXT_OP mova, r0, r2, r1 + DC_ADD_MMXEXT_OP mova, r0, r2, r3 + RET +%else +cglobal h264_idct_dc_add_8, 2, 3, 0 + movsx r2, word [r1] + mov dword [r1], 0 + mov r1, r2m + DC_ADD_MMXEXT_INIT r2, r1 + DC_ADD_MMXEXT_OP movh, r0, r1, r2 RET +; ff_h264_idct8_dc_add_mmxext(uint8_t *dst, int16_t *block, int stride) +cglobal h264_idct8_dc_add_8, 2, 3, 0 + movsx r2, word [r1] + mov dword [r1], 0 + mov r1, r2m + DC_ADD_MMXEXT_INIT r2, r1 + DC_ADD_MMXEXT_OP mova, r0, r1, r2 + lea r0, [r0+r1*4] + DC_ADD_MMXEXT_OP mova, r0, r1, r2 + RET +%endif + INIT_MMX mmx ; ff_h264_idct_add16_mmx(uint8_t *dst, const int *block_offset, ; int16_t *block, int stride, const uint8_t nnzc[6*8]) @@ -343,7 +390,7 @@ cglobal h264_idct8_add4_8, 5, 7 + npicregs, 0, dst, block_offset, block, stride, add word [r2], 32 IDCT8_ADD_MMX_START r2 , rsp IDCT8_ADD_MMX_START r2+8, rsp+64 - IDCT8_ADD_MMX_END r6 , rsp, r3 + IDCT8_ADD_MMX_END r6 , rsp, r3, r2 mov r6d, dword [r1+r5*4] lea r6, [r0+r6+4] IDCT8_ADD_MMX_END r6 , rsp+8, r3 @@ -373,7 +420,8 @@ cglobal h264_idct_add16_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride movsx r6, word [r2] test r6, r6 jz .no_dc - DC_ADD_MMXEXT_INIT r2, r3, r6 + mov word [r2], 0 + DC_ADD_MMXEXT_INIT r6, r3 %if ARCH_X86_64 == 0 %define dst2q r1 %define dst2d r1d @@ -450,7 +498,8 @@ cglobal h264_idct_add16intra_8, 5, 8 + npicregs, 0, dst1, block_offset, block, s movsx r6, word [r2] test r6, r6 jz .skipblock - DC_ADD_MMXEXT_INIT r2, r3, r6 + mov word [r2], 0 + DC_ADD_MMXEXT_INIT r6, r3 %if ARCH_X86_64 == 0 %define dst2q r1 %define dst2d r1d @@ -489,7 +538,8 @@ cglobal h264_idct8_add4_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride movsx r6, word [r2] test r6, r6 jz .no_dc - DC_ADD_MMXEXT_INIT r2, r3, r6 + mov word [r2], 0 + DC_ADD_MMXEXT_INIT r6, r3 %if ARCH_X86_64 == 0 %define dst2q r1 %define dst2d r1d @@ -515,7 +565,7 @@ cglobal h264_idct8_add4_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride add word [r2], 32 IDCT8_ADD_MMX_START r2 , rsp IDCT8_ADD_MMX_START r2+8, rsp+64 - IDCT8_ADD_MMX_END r6 , rsp, r3 + IDCT8_ADD_MMX_END r6 , rsp, r3, r2 mov r6d, dword [r1+r5*4] lea r6, [r0+r6+4] IDCT8_ADD_MMX_END r6 , rsp+8, r3 @@ -547,7 +597,8 @@ cglobal h264_idct8_add4_8, 5, 8 + npicregs, 10, dst1, block_offset, block, strid test r6, r6 jz .no_dc INIT_MMX cpuname - DC_ADD_MMXEXT_INIT r2, r3, r6 + mov word [r2], 0 + DC_ADD_MMXEXT_INIT r6, r3 %if ARCH_X86_64 == 0 %define dst2q r1 %define dst2d r1d @@ -650,7 +701,8 @@ h264_idct_add8_mmxext_plane: movsx r6, word [r2] test r6, r6 jz .skipblock - DC_ADD_MMXEXT_INIT r2, r3, r6 + mov word [r2], 0 + DC_ADD_MMXEXT_INIT r6, r3 %if ARCH_X86_64 mov r0d, dword [r1+r5*4] add r0, [dst2q] @@ -693,7 +745,9 @@ cglobal h264_idct_add8_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, ; r0 = uint8_t *dst, r2 = int16_t *block, r3 = int stride, r6=clobbered h264_idct_dc_add8_mmxext: movd m0, [r2 ] ; 0 0 X D + mov word [r2+ 0], 0 punpcklwd m0, [r2+32] ; x X d D + mov word [r2+32], 0 paddsw m0, [pw_32] psraw m0, 6 punpcklwd m0, m0 ; d d D D @@ -723,6 +777,10 @@ h264_add8x4_idct_sse2: paddw m0, [pw_32] IDCT4_1D w,0,1,2,3,4,5 pxor m7, m7 + mova [r2+ 0], m7 + mova [r2+16], m7 + mova [r2+32], m7 + mova [r2+48], m7 STORE_DIFFx2 m0, m1, m4, m5, m7, 6, r0, r3 lea r0, [r0+r3*2] STORE_DIFFx2 m2, m3, m4, m5, m7, 6, r0, r3 diff --git a/libavcodec/x86/h264_idct_10bit.asm b/libavcodec/x86/h264_idct_10bit.asm index 51965f0f9f..4e51d2b5d0 100644 --- a/libavcodec/x86/h264_idct_10bit.asm +++ b/libavcodec/x86/h264_idct_10bit.asm @@ -66,6 +66,10 @@ SECTION .text paddd m0, [pd_32] IDCT4_1D d,0,1,2,3,4,5 pxor m5, m5 + mova [%2+ 0], m5 + mova [%2+16], m5 + mova [%2+32], m5 + mova [%2+48], m5 STORE_DIFFx2 m0, m1, m4, m5, %1, %3 lea %1, [%1+%3*2] STORE_DIFFx2 m2, m3, m4, m5, %1, %3 @@ -98,6 +102,10 @@ add4x4_idct %+ SUFFIX: paddd m0, [pd_32] IDCT4_1D d,0,1,2,3,4,5 pxor m5, m5 + mova [r2+ 0], m5 + mova [r2+16], m5 + mova [r2+32], m5 + mova [r2+48], m5 STORE_DIFFx2 m0, m1, m4, m5, r5, r3 lea r5, [r5+r3*2] STORE_DIFFx2 m2, m3, m4, m5, r5, r3 @@ -181,6 +189,7 @@ IDCT_ADD16_10 INIT_MMX mmxext cglobal h264_idct_dc_add_10,3,3 movd m0, [r1] + mov dword [r1], 0 paddd m0, [pd_32] psrad m0, 6 lea r1, [r2*3] @@ -193,11 +202,11 @@ cglobal h264_idct_dc_add_10,3,3 ; void h264_idct8_dc_add(pixel *dst, dctcoef *block, int stride) ;----------------------------------------------------------------------------- %macro IDCT8_DC_ADD 0 -cglobal h264_idct8_dc_add_10,3,3,7 - mov r1d, [r1] - add r1, 32 - sar r1, 6 - movd m0, r1d +cglobal h264_idct8_dc_add_10,3,4,7 + movd m0, [r1] + mov dword[r1], 0 + paddd m0, [pd_32] + psrad m0, 6 lea r1, [r2*3] SPLATW m0, m0, 0 mova m6, [pw_pixel_max] @@ -247,6 +256,8 @@ idct_dc_add %+ SUFFIX: add r5, r0 movq m0, [r2+ 0] movhps m0, [r2+64] + mov dword [r2+ 0], 0 + mov dword [r2+64], 0 paddd m0, [pd_32] psrad m0, 6 pshufhw m0, m0, 0 @@ -461,6 +472,22 @@ h264_idct8_add1_10 %+ SUFFIX: packssdw m8, m0 paddsw m8, [r0] pxor m0, m0 + mova [r1+ 0], m0 + mova [r1+ 16], m0 + mova [r1+ 32], m0 + mova [r1+ 48], m0 + mova [r1+ 64], m0 + mova [r1+ 80], m0 + mova [r1+ 96], m0 + mova [r1+112], m0 + mova [r1+128], m0 + mova [r1+144], m0 + mova [r1+160], m0 + mova [r1+176], m0 + mova [r1+192], m0 + mova [r1+208], m0 + mova [r1+224], m0 + mova [r1+240], m0 CLIPW m8, m0, [pw_pixel_max] mova [r0], m8 mova m8, [pw_pixel_max] @@ -480,6 +507,22 @@ h264_idct8_add1_10 %+ SUFFIX: lea r3, [r0+8] IDCT8_ADD_SSE_END r0, rsp, r2 IDCT8_ADD_SSE_END r3, rsp+16, r2 + mova [r1+ 0], m7 + mova [r1+ 16], m7 + mova [r1+ 32], m7 + mova [r1+ 48], m7 + mova [r1+ 64], m7 + mova [r1+ 80], m7 + mova [r1+ 96], m7 + mova [r1+112], m7 + mova [r1+128], m7 + mova [r1+144], m7 + mova [r1+160], m7 + mova [r1+176], m7 + mova [r1+192], m7 + mova [r1+208], m7 + mova [r1+224], m7 + mova [r1+240], m7 %endif ; ARCH_X86_64 add rsp, pad -- cgit v1.2.3