From b57e38f52cc3f31a27105c28887d57cd6812c3eb Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Wed, 28 Oct 2015 15:38:22 +0100 Subject: ac3dsp: x86: Replace inline asm for in-decoder downmixing with standalone asm Adds a wrapper function for downmixing which detects channel count changes and updates the selected downmix function accordingly. Simplification and porting to current x86inc infrastructure by Diego Biurrun. Signed-off-by: Diego Biurrun --- libavcodec/ac3dec.c | 8 +- libavcodec/ac3dsp.c | 112 +++++++++++++++-------- libavcodec/ac3dsp.h | 9 +- libavcodec/x86/Makefile | 3 +- libavcodec/x86/ac3dsp_downmix.asm | 187 ++++++++++++++++++++++++++++++++++++++ libavcodec/x86/ac3dsp_init.c | 174 +++++++++-------------------------- 6 files changed, 316 insertions(+), 177 deletions(-) create mode 100644 libavcodec/x86/ac3dsp_downmix.asm (limited to 'libavcodec') diff --git a/libavcodec/ac3dec.c b/libavcodec/ac3dec.c index aba31195fb..ad50552ec1 100644 --- a/libavcodec/ac3dec.c +++ b/libavcodec/ac3dec.c @@ -1328,19 +1328,19 @@ static int decode_audio_block(AC3DecodeContext *s, int blk) do_imdct(s, s->channels); if (downmix_output) { - s->ac3dsp.downmix(s->outptr, s->downmix_coeffs, + ff_ac3dsp_downmix(&s->ac3dsp, s->outptr, s->downmix_coeffs, s->out_channels, s->fbw_channels, 256); } } else { if (downmix_output) { - s->ac3dsp.downmix(s->xcfptr + 1, s->downmix_coeffs, + ff_ac3dsp_downmix(&s->ac3dsp, s->xcfptr + 1, s->downmix_coeffs, s->out_channels, s->fbw_channels, 256); } if (downmix_output && !s->downmixed) { s->downmixed = 1; - s->ac3dsp.downmix(s->dlyptr, s->downmix_coeffs, s->out_channels, - s->fbw_channels, 128); + ff_ac3dsp_downmix(&s->ac3dsp, s->dlyptr, s->downmix_coeffs, + s->out_channels, s->fbw_channels, 128); } do_imdct(s, s->out_channels); diff --git a/libavcodec/ac3dsp.c b/libavcodec/ac3dsp.c index 8eae5ad866..440bd1a991 100644 --- a/libavcodec/ac3dsp.c +++ b/libavcodec/ac3dsp.c @@ -171,49 +171,53 @@ static void ac3_extract_exponents_c(uint8_t *exp, int32_t *coef, int nb_coefs) } } -static void ac3_downmix_c(float **samples, float **matrix, - int out_ch, int in_ch, int len) +static void ac3_downmix_5_to_2_symmetric_c(float **samples, float **matrix, + int len) { - int **matrix_cmp = (int **)matrix; - int i, j; + int i; float v0, v1; + float front_mix = matrix[0][0]; + float center_mix = matrix[0][1]; + float surround_mix = matrix[0][3]; - if (in_ch == 5 && out_ch == 2 && - !(matrix_cmp[1][0] | matrix_cmp[0][2] | - matrix_cmp[1][3] | matrix_cmp[0][4] | - (matrix_cmp[0][1] ^ matrix_cmp[1][1]) | - (matrix_cmp[0][0] ^ matrix_cmp[1][2]))) { - float front_mix = matrix[0][0]; - float center_mix = matrix[0][1]; - float surround_mix = matrix[0][3]; + for (i = 0; i < len; i++) { + v0 = samples[0][i] * front_mix + + samples[1][i] * center_mix + + samples[3][i] * surround_mix; - for (i = 0; i < len; i++) { - v0 = samples[0][i] * front_mix + - samples[1][i] * center_mix + - samples[3][i] * surround_mix; + v1 = samples[1][i] * center_mix + + samples[2][i] * front_mix + + samples[4][i] * surround_mix; - v1 = samples[1][i] * center_mix + - samples[2][i] * front_mix + - samples[4][i] * surround_mix; + samples[0][i] = v0; + samples[1][i] = v1; + } +} - samples[0][i] = v0; - samples[1][i] = v1; - } - } else if (in_ch == 5 && out_ch == 1 && - matrix_cmp[0][0] == matrix_cmp[0][2] && - matrix_cmp[0][3] == matrix_cmp[0][4]) { - float front_mix = matrix[0][0]; - float center_mix = matrix[0][1]; - float surround_mix = matrix[0][3]; +static void ac3_downmix_5_to_1_symmetric_c(float **samples, float **matrix, + int len) +{ + int i; + float front_mix = matrix[0][0]; + float center_mix = matrix[0][1]; + float surround_mix = matrix[0][3]; - for (i = 0; i < len; i++) { - samples[0][i] = samples[0][i] * front_mix + - samples[1][i] * center_mix + - samples[2][i] * front_mix + - samples[3][i] * surround_mix + - samples[4][i] * surround_mix; - } - } else if (out_ch == 2) { + for (i = 0; i < len; i++) { + samples[0][i] = samples[0][i] * front_mix + + samples[1][i] * center_mix + + samples[2][i] * front_mix + + samples[3][i] * surround_mix + + samples[4][i] * surround_mix; + } +} + +static void ac3_downmix_c(float **samples, float **matrix, + int out_ch, int in_ch, int len) +{ + int i, j; + float v0, v1; + + if (out_ch == 2) { for (i = 0; i < len; i++) { v0 = v1 = 0.0f; for (j = 0; j < in_ch; j++) { @@ -246,6 +250,38 @@ static void apply_window_int16_c(int16_t *output, const int16_t *input, } } +void ff_ac3dsp_downmix(AC3DSPContext *c, float **samples, float **matrix, + int out_ch, int in_ch, int len) +{ + if (c->in_channels != in_ch || c->out_channels != out_ch) { + int **matrix_cmp = (int **)matrix; + + c->in_channels = in_ch; + c->out_channels = out_ch; + c->downmix = NULL; + + if (in_ch == 5 && out_ch == 2 && + !(matrix_cmp[1][0] | matrix_cmp[0][2] | + matrix_cmp[1][3] | matrix_cmp[0][4] | + (matrix_cmp[0][1] ^ matrix_cmp[1][1]) | + (matrix_cmp[0][0] ^ matrix_cmp[1][2]))) { + c->downmix = ac3_downmix_5_to_2_symmetric_c; + } else if (in_ch == 5 && out_ch == 1 && + matrix_cmp[0][0] == matrix_cmp[0][2] && + matrix_cmp[0][3] == matrix_cmp[0][4]) { + c->downmix = ac3_downmix_5_to_1_symmetric_c; + } + + if (ARCH_X86) + ff_ac3dsp_set_downmix_x86(c); + } + + if (c->downmix) + c->downmix(samples, matrix, len); + else + ac3_downmix_c(samples, matrix, out_ch, in_ch, len); +} + av_cold void ff_ac3dsp_init(AC3DSPContext *c, int bit_exact) { c->ac3_exponent_min = ac3_exponent_min_c; @@ -257,7 +293,9 @@ av_cold void ff_ac3dsp_init(AC3DSPContext *c, int bit_exact) c->update_bap_counts = ac3_update_bap_counts_c; c->compute_mantissa_size = ac3_compute_mantissa_size_c; c->extract_exponents = ac3_extract_exponents_c; - c->downmix = ac3_downmix_c; + c->in_channels = 0; + c->out_channels = 0; + c->downmix = NULL; c->apply_window_int16 = apply_window_int16_c; if (ARCH_ARM) diff --git a/libavcodec/ac3dsp.h b/libavcodec/ac3dsp.h index cdce21a865..c33a0bee84 100644 --- a/libavcodec/ac3dsp.h +++ b/libavcodec/ac3dsp.h @@ -126,8 +126,9 @@ typedef struct AC3DSPContext { void (*extract_exponents)(uint8_t *exp, int32_t *coef, int nb_coefs); - void (*downmix)(float **samples, float **matrix, int out_ch, - int in_ch, int len); + int out_channels; + int in_channels; + void (*downmix)(float **samples, float **matrix, int len); /** * Apply symmetric window in 16-bit fixed-point. @@ -148,4 +149,8 @@ void ff_ac3dsp_init (AC3DSPContext *c, int bit_exact); void ff_ac3dsp_init_arm(AC3DSPContext *c, int bit_exact); void ff_ac3dsp_init_x86(AC3DSPContext *c, int bit_exact); +void ff_ac3dsp_downmix(AC3DSPContext *c, float **samples, float **matrix, + int out_ch, int in_ch, int len); +void ff_ac3dsp_set_downmix_x86(AC3DSPContext *c); + #endif /* AVCODEC_AC3DSP_H */ diff --git a/libavcodec/x86/Makefile b/libavcodec/x86/Makefile index 872b7faddb..4aae594bd8 100644 --- a/libavcodec/x86/Makefile +++ b/libavcodec/x86/Makefile @@ -71,7 +71,8 @@ MMX-OBJS-$(CONFIG_MPEG4_DECODER) += x86/xvididct_mmx.o \ x86/xvididct_sse2.o # subsystems -YASM-OBJS-$(CONFIG_AC3DSP) += x86/ac3dsp.o +YASM-OBJS-$(CONFIG_AC3DSP) += x86/ac3dsp.o \ + x86/ac3dsp_downmix.o YASM-OBJS-$(CONFIG_AUDIODSP) += x86/audiodsp.o YASM-OBJS-$(CONFIG_BSWAPDSP) += x86/bswapdsp.o YASM-OBJS-$(CONFIG_DCT) += x86/dct32.o diff --git a/libavcodec/x86/ac3dsp_downmix.asm b/libavcodec/x86/ac3dsp_downmix.asm new file mode 100644 index 0000000000..b085035ce3 --- /dev/null +++ b/libavcodec/x86/ac3dsp_downmix.asm @@ -0,0 +1,187 @@ +;***************************************************************************** +;* x86-optimized AC-3 downmixing +;* Copyright (c) 2012 Justin Ruggles +;* +;* This file is part of Libav. +;* +;* Libav is free software; you can redistribute it and/or +;* modify it under the terms of the GNU Lesser General Public +;* License as published by the Free Software Foundation; either +;* version 2.1 of the License, or (at your option) any later version. +;* +;* Libav is distributed in the hope that it will be useful, +;* but WITHOUT ANY WARRANTY; without even the implied warranty of +;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;* Lesser General Public License for more details. +;* +;* You should have received a copy of the GNU Lesser General Public +;* License along with Libav; if not, write to the Free Software +;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +;****************************************************************************** + +;****************************************************************************** +;* This is based on the channel mixing asm in libavresample, but it is +;* simplified for only float coefficients and only 3 to 6 channels. +;****************************************************************************** + +%include "libavutil/x86/x86util.asm" + +SECTION .text + +;----------------------------------------------------------------------------- +; functions to downmix from 3 to 6 channels to mono or stereo +; void ff_ac3_downmix_*(float **samples, float **matrix, int len); +;----------------------------------------------------------------------------- + +%macro AC3_DOWNMIX 2 ; %1 = in channels, %2 = out channels +; define some names to make the code clearer +%assign in_channels %1 +%assign out_channels %2 +%assign stereo out_channels - 1 + +; determine how many matrix elements must go on the stack vs. mmregs +%assign matrix_elements in_channels * out_channels +%if stereo + %assign needed_mmregs 4 +%else + %assign needed_mmregs 3 +%endif +%assign matrix_elements_mm num_mmregs - needed_mmregs +%if matrix_elements < matrix_elements_mm + %assign matrix_elements_mm matrix_elements +%endif +%assign total_mmregs needed_mmregs+matrix_elements_mm +%if matrix_elements_mm < matrix_elements + %assign matrix_elements_stack matrix_elements - matrix_elements_mm +%else + %assign matrix_elements_stack 0 +%endif + +cglobal ac3_downmix_%1_to_%2, 3,in_channels+1,total_mmregs,0-matrix_elements_stack*mmsize, src0, src1, len, src2, src3, src4, src5 + +; load matrix pointers +%define matrix0q r1q +%define matrix1q r3q +%if stereo + mov matrix1q, [matrix0q+gprsize] +%endif + mov matrix0q, [matrix0q] + +; define matrix coeff names +%assign %%i 0 +%assign %%j needed_mmregs +%rep in_channels + %if %%i >= matrix_elements_mm + CAT_XDEFINE mx_stack_0_, %%i, 1 + CAT_XDEFINE mx_0_, %%i, [rsp+(%%i-matrix_elements_mm)*mmsize] + %else + CAT_XDEFINE mx_stack_0_, %%i, 0 + CAT_XDEFINE mx_0_, %%i, m %+ %%j + %assign %%j %%j+1 + %endif + %assign %%i %%i+1 +%endrep +%if stereo +%assign %%i 0 +%rep in_channels + %if in_channels + %%i >= matrix_elements_mm + CAT_XDEFINE mx_stack_1_, %%i, 1 + CAT_XDEFINE mx_1_, %%i, [rsp+(in_channels+%%i-matrix_elements_mm)*mmsize] + %else + CAT_XDEFINE mx_stack_1_, %%i, 0 + CAT_XDEFINE mx_1_, %%i, m %+ %%j + %assign %%j %%j+1 + %endif + %assign %%i %%i+1 +%endrep +%endif + +; load/splat matrix coeffs +%assign %%i 0 +%rep in_channels + %if mx_stack_0_ %+ %%i + VBROADCASTSS m0, [matrix0q+4*%%i] + mova mx_0_ %+ %%i, m0 + %else + VBROADCASTSS mx_0_ %+ %%i, [matrix0q+4*%%i] + %endif + %if stereo + %if mx_stack_1_ %+ %%i + VBROADCASTSS m0, [matrix1q+4*%%i] + mova mx_1_ %+ %%i, m0 + %else + VBROADCASTSS mx_1_ %+ %%i, [matrix1q+4*%%i] + %endif + %endif + %assign %%i %%i+1 +%endrep + + lea lenq, [4*r2d] + ; load channel pointers to registers +%assign %%i 1 +%rep (in_channels - 1) + mov src %+ %%i %+ q, [src0q+%%i*gprsize] + add src %+ %%i %+ q, lenq + %assign %%i %%i+1 +%endrep + mov src0q, [src0q] + add src0q, lenq + neg lenq +.loop: + %if stereo || mx_stack_0_0 + mova m0, [src0q+lenq] + %endif + %if stereo + mulps m1, m0, mx_1_0 + %endif + %if stereo || mx_stack_0_0 + mulps m0, m0, mx_0_0 + %else + mulps m0, mx_0_0, [src0q+lenq] + %endif +%assign %%i 1 +%rep (in_channels - 1) + %define src_ptr src %+ %%i %+ q + ; avoid extra load for mono if matrix is in a mm register + %if stereo || mx_stack_0_ %+ %%i + mova m2, [src_ptr+lenq] + %endif + %if stereo + FMULADD_PS m1, m2, mx_1_ %+ %%i, m1, m3 + %endif + %if stereo || mx_stack_0_ %+ %%i + FMULADD_PS m0, m2, mx_0_ %+ %%i, m0, m2 + %else + FMULADD_PS m0, mx_0_ %+ %%i, [src_ptr+lenq], m0, m1 + %endif + %assign %%i %%i+1 +%endrep + mova [src0q+lenq], m0 + %if stereo + mova [src1q+lenq], m1 + %endif + + add lenq, mmsize + jl .loop + RET +%endmacro + +%macro AC3_DOWNMIX_FUNCS 0 +%assign %%i 3 +%rep 4 + INIT_XMM sse + AC3_DOWNMIX %%i, 1 + AC3_DOWNMIX %%i, 2 + INIT_YMM avx + AC3_DOWNMIX %%i, 1 + AC3_DOWNMIX %%i, 2 + %if HAVE_FMA3_EXTERNAL + INIT_YMM fma3 + AC3_DOWNMIX %%i, 1 + AC3_DOWNMIX %%i, 2 + %endif + %assign %%i %%i+1 +%endrep +%endmacro + +AC3_DOWNMIX_FUNCS diff --git a/libavcodec/x86/ac3dsp_init.c b/libavcodec/x86/ac3dsp_init.c index 9036389296..6d049b37cf 100644 --- a/libavcodec/x86/ac3dsp_init.c +++ b/libavcodec/x86/ac3dsp_init.c @@ -63,135 +63,6 @@ void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input, void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input, const int16_t *window, unsigned int len); -#if HAVE_SSE_INLINE && HAVE_7REGS - -#define IF1(x) x -#define IF0(x) - -#define MIX5(mono, stereo) \ - __asm__ volatile ( \ - "movss 0(%1), %%xmm5 \n" \ - "movss 4(%1), %%xmm6 \n" \ - "movss 12(%1), %%xmm7 \n" \ - "shufps $0, %%xmm5, %%xmm5 \n" \ - "shufps $0, %%xmm6, %%xmm6 \n" \ - "shufps $0, %%xmm7, %%xmm7 \n" \ - "1: \n" \ - "movaps (%0, %2), %%xmm0 \n" \ - "movaps (%0, %3), %%xmm1 \n" \ - "movaps (%0, %4), %%xmm2 \n" \ - "movaps (%0, %5), %%xmm3 \n" \ - "movaps (%0, %6), %%xmm4 \n" \ - "mulps %%xmm5, %%xmm0 \n" \ - "mulps %%xmm6, %%xmm1 \n" \ - "mulps %%xmm5, %%xmm2 \n" \ - "mulps %%xmm7, %%xmm3 \n" \ - "mulps %%xmm7, %%xmm4 \n" \ - stereo("addps %%xmm1, %%xmm0 \n") \ - "addps %%xmm1, %%xmm2 \n" \ - "addps %%xmm3, %%xmm0 \n" \ - "addps %%xmm4, %%xmm2 \n" \ - mono("addps %%xmm2, %%xmm0 \n") \ - "movaps %%xmm0, (%0, %2) \n" \ - stereo("movaps %%xmm2, (%0, %3) \n") \ - "add $16, %0 \n" \ - "jl 1b \n" \ - : "+&r"(i) \ - : "r"(matrix[0]), \ - "r"(samples[0] + len), \ - "r"(samples[1] + len), \ - "r"(samples[2] + len), \ - "r"(samples[3] + len), \ - "r"(samples[4] + len) \ - : XMM_CLOBBERS("%xmm0", "%xmm1", "%xmm2", "%xmm3", \ - "%xmm4", "%xmm5", "%xmm6", "%xmm7",) \ - "memory" \ - ); - -#define MIX_MISC(stereo) \ - __asm__ volatile ( \ - "mov %5, %2 \n" \ - "1: \n" \ - "mov -%c7(%6, %2, %c8), %3 \n" \ - "movaps (%3, %0), %%xmm0 \n" \ - stereo("movaps %%xmm0, %%xmm1 \n") \ - "mulps %%xmm4, %%xmm0 \n" \ - stereo("mulps %%xmm5, %%xmm1 \n") \ - "2: \n" \ - "mov (%6, %2, %c8), %1 \n" \ - "movaps (%1, %0), %%xmm2 \n" \ - stereo("movaps %%xmm2, %%xmm3 \n") \ - "mulps (%4, %2, 8), %%xmm2 \n" \ - stereo("mulps 16(%4, %2, 8), %%xmm3 \n") \ - "addps %%xmm2, %%xmm0 \n" \ - stereo("addps %%xmm3, %%xmm1 \n") \ - "add $4, %2 \n" \ - "jl 2b \n" \ - "mov %5, %2 \n" \ - stereo("mov (%6, %2, %c8), %1 \n") \ - "movaps %%xmm0, (%3, %0) \n" \ - stereo("movaps %%xmm1, (%1, %0) \n") \ - "add $16, %0 \n" \ - "jl 1b \n" \ - : "+&r"(i), "=&r"(j), "=&r"(k), "=&r"(m) \ - : "r"(matrix_simd + in_ch), \ - "g"((intptr_t) - 4 * (in_ch - 1)), \ - "r"(samp + in_ch), \ - "i"(sizeof(float *)), "i"(sizeof(float *)/4) \ - : "memory" \ - ); - -static void ac3_downmix_sse(float **samples, float **matrix, - int out_ch, int in_ch, int len) -{ - int **matrix_cmp = (int **)matrix; - intptr_t i, j, k, m; - - i = -len * sizeof(float); - if (in_ch == 5 && out_ch == 2 && - !(matrix_cmp[1][0] | matrix_cmp[0][2] | - matrix_cmp[1][3] | matrix_cmp[0][4] | - (matrix_cmp[0][1] ^ matrix_cmp[1][1]) | - (matrix_cmp[0][0] ^ matrix_cmp[1][2]))) { - MIX5(IF0, IF1); - } else if (in_ch == 5 && out_ch == 1 && - matrix_cmp[0][0] == matrix_cmp[0][2] && - matrix_cmp[0][3] == matrix_cmp[0][4]) { - MIX5(IF1, IF0); - } else { - DECLARE_ALIGNED(16, float, matrix_simd)[AC3_MAX_CHANNELS][2][4]; - float *samp[AC3_MAX_CHANNELS]; - - for (j = 0; j < in_ch; j++) - samp[j] = samples[j] + len; - - j = 2 * in_ch * sizeof(float); - k = in_ch * sizeof(float); - __asm__ volatile ( - "1: \n" - "sub $4, %1 \n" - "sub $8, %0 \n" - "movss (%3, %1), %%xmm4 \n" - "movss (%4, %1), %%xmm5 \n" - "shufps $0, %%xmm4, %%xmm4 \n" - "shufps $0, %%xmm5, %%xmm5 \n" - "movaps %%xmm4, (%2, %0, 4) \n" - "movaps %%xmm5, 16(%2, %0, 4) \n" - "jg 1b \n" - : "+&r"(j), "+&r"(k) - : "r"(matrix_simd), "r"(matrix[0]), "r"(matrix[1]) - : "memory" - ); - if (out_ch == 2) { - MIX_MISC(IF1); - } else { - MIX_MISC(IF0); - } - } -} - -#endif /* HAVE_SSE_INLINE && HAVE_7REGS */ - av_cold void ff_ac3dsp_init_x86(AC3DSPContext *c, int bit_exact) { int cpu_flags = av_get_cpu_flags(); @@ -247,10 +118,47 @@ av_cold void ff_ac3dsp_init_x86(AC3DSPContext *c, int bit_exact) c->apply_window_int16 = ff_apply_window_int16_ssse3; } } +} + +#define DOWNMIX_FUNC_OPT(ch, opt) \ +void ff_ac3_downmix_ ## ch ## _to_1_ ## opt(float **samples, \ + float **matrix, int len); \ +void ff_ac3_downmix_ ## ch ## _to_2_ ## opt(float **samples, \ + float **matrix, int len); + +#define DOWNMIX_FUNCS(opt) \ + DOWNMIX_FUNC_OPT(3, opt) \ + DOWNMIX_FUNC_OPT(4, opt) \ + DOWNMIX_FUNC_OPT(5, opt) \ + DOWNMIX_FUNC_OPT(6, opt) + +DOWNMIX_FUNCS(sse) +DOWNMIX_FUNCS(avx) +DOWNMIX_FUNCS(fma3) + +void ff_ac3dsp_set_downmix_x86(AC3DSPContext *c) +{ + int cpu_flags = av_get_cpu_flags(); + +#define SET_DOWNMIX(ch, suf, SUF) \ + if (ch == c->in_channels) { \ + if (EXTERNAL_ ## SUF (cpu_flags)) { \ + if (c->out_channels == 1) \ + c->downmix = ff_ac3_downmix_ ## ch ## _to_1_ ## suf; \ + else \ + c->downmix = ff_ac3_downmix_ ## ch ## _to_2_ ## suf; \ + } \ + } + +#define SET_DOWNMIX_ALL(suf, SUF) \ + SET_DOWNMIX(3, suf, SUF) \ + SET_DOWNMIX(4, suf, SUF) \ + SET_DOWNMIX(5, suf, SUF) \ + SET_DOWNMIX(6, suf, SUF) -#if HAVE_SSE_INLINE && HAVE_7REGS - if (INLINE_SSE(cpu_flags)) { - c->downmix = ac3_downmix_sse; + SET_DOWNMIX_ALL(sse, SSE) + if (!(cpu_flags & AV_CPU_FLAG_AVXSLOW)) { + SET_DOWNMIX_ALL(avx, AVX) + SET_DOWNMIX_ALL(fma3, FMA3) } -#endif } -- cgit v1.2.3