/* * Copyright (c) 2021 Loongson Technology Corporation Limited * Contributed by Hao Chen * * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "libavcodec/vp9dsp.h" #include "libavutil/loongarch/loongson_intrinsics.h" #include "vp9dsp_loongarch.h" #define LSX_ST_8(_dst0, _dst1, _dst2, _dst3, _dst4, \ _dst5, _dst6, _dst7, _dst, _stride, \ _stride2, _stride3, _stride4) \ { \ __lsx_vst(_dst0, _dst, 0); \ __lsx_vstx(_dst1, _dst, _stride); \ __lsx_vstx(_dst2, _dst, _stride2); \ __lsx_vstx(_dst3, _dst, _stride3); \ _dst += _stride4; \ __lsx_vst(_dst4, _dst, 0); \ __lsx_vstx(_dst5, _dst, _stride); \ __lsx_vstx(_dst6, _dst, _stride2); \ __lsx_vstx(_dst7, _dst, _stride3); \ } #define LSX_ST_8X16(_dst0, _dst1, _dst2, _dst3, _dst4, \ _dst5, _dst6, _dst7, _dst, _stride) \ { \ __lsx_vst(_dst0, _dst, 0); \ __lsx_vst(_dst0, _dst, 16); \ _dst += _stride; \ __lsx_vst(_dst1, _dst, 0); \ __lsx_vst(_dst1, _dst, 16); \ _dst += _stride; \ __lsx_vst(_dst2, _dst, 0); \ __lsx_vst(_dst2, _dst, 16); \ _dst += _stride; \ __lsx_vst(_dst3, _dst, 0); \ __lsx_vst(_dst3, _dst, 16); \ _dst += _stride; \ __lsx_vst(_dst4, _dst, 0); \ __lsx_vst(_dst4, _dst, 16); \ _dst += _stride; \ __lsx_vst(_dst5, _dst, 0); \ __lsx_vst(_dst5, _dst, 16); \ _dst += _stride; \ __lsx_vst(_dst6, _dst, 0); \ __lsx_vst(_dst6, _dst, 16); \ _dst += _stride; \ __lsx_vst(_dst7, _dst, 0); \ __lsx_vst(_dst7, _dst, 16); \ _dst += _stride; \ } void ff_vert_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *left, const uint8_t *src) { __m128i src0; ptrdiff_t stride2 = dst_stride << 1; ptrdiff_t stride3 = stride2 + dst_stride; ptrdiff_t stride4 = stride2 << 1; src0 = __lsx_vld(src, 0); LSX_ST_8(src0, src0, src0, src0, src0, src0, src0, src0, dst, dst_stride, stride2, stride3, stride4); dst += stride4; LSX_ST_8(src0, src0, src0, src0, src0, src0, src0, src0, dst, dst_stride, stride2, stride3, stride4); } void ff_vert_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *left, const uint8_t *src) { uint32_t row; __m128i src0, src1; DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); for (row = 32; row--;) { __lsx_vst(src0, dst, 0); __lsx_vst(src1, dst, 16); dst += dst_stride; } } void ff_hor_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src, const uint8_t *top) { __m128i src0, src1, src2, src3, src4, src5, src6, src7; __m128i src8, src9, src10, src11, src12, src13, src14, src15; ptrdiff_t stride2 = dst_stride << 1; ptrdiff_t stride3 = stride2 + dst_stride; ptrdiff_t stride4 = stride2 << 1; src15 = __lsx_vldrepl_b(src, 0); src14 = __lsx_vldrepl_b(src, 1); src13 = __lsx_vldrepl_b(src, 2); src12 = __lsx_vldrepl_b(src, 3); src11 = __lsx_vldrepl_b(src, 4); src10 = __lsx_vldrepl_b(src, 5); src9 = __lsx_vldrepl_b(src, 6); src8 = __lsx_vldrepl_b(src, 7); src7 = __lsx_vldrepl_b(src, 8); src6 = __lsx_vldrepl_b(src, 9); src5 = __lsx_vldrepl_b(src, 10); src4 = __lsx_vldrepl_b(src, 11); src3 = __lsx_vldrepl_b(src, 12); src2 = __lsx_vldrepl_b(src, 13); src1 = __lsx_vldrepl_b(src, 14); src0 = __lsx_vldrepl_b(src, 15); LSX_ST_8(src0, src1, src2, src3, src4, src5, src6, src7, dst, dst_stride, stride2, stride3, stride4); dst += stride4; LSX_ST_8(src8, src9, src10, src11, src12, src13, src14, src15, dst, dst_stride, stride2, stride3, stride4); } void ff_hor_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src, const uint8_t *top) { __m128i src0, src1, src2, src3, src4, src5, src6, src7; __m128i src8, src9, src10, src11, src12, src13, src14, src15; __m128i src16, src17, src18, src19, src20, src21, src22, src23; __m128i src24, src25, src26, src27, src28, src29, src30, src31; src31 = __lsx_vldrepl_b(src, 0); src30 = __lsx_vldrepl_b(src, 1); src29 = __lsx_vldrepl_b(src, 2); src28 = __lsx_vldrepl_b(src, 3); src27 = __lsx_vldrepl_b(src, 4); src26 = __lsx_vldrepl_b(src, 5); src25 = __lsx_vldrepl_b(src, 6); src24 = __lsx_vldrepl_b(src, 7); src23 = __lsx_vldrepl_b(src, 8); src22 = __lsx_vldrepl_b(src, 9); src21 = __lsx_vldrepl_b(src, 10); src20 = __lsx_vldrepl_b(src, 11); src19 = __lsx_vldrepl_b(src, 12); src18 = __lsx_vldrepl_b(src, 13); src17 = __lsx_vldrepl_b(src, 14); src16 = __lsx_vldrepl_b(src, 15); src15 = __lsx_vldrepl_b(src, 16); src14 = __lsx_vldrepl_b(src, 17); src13 = __lsx_vldrepl_b(src, 18); src12 = __lsx_vldrepl_b(src, 19); src11 = __lsx_vldrepl_b(src, 20); src10 = __lsx_vldrepl_b(src, 21); src9 = __lsx_vldrepl_b(src, 22); src8 = __lsx_vldrepl_b(src, 23); src7 = __lsx_vldrepl_b(src, 24); src6 = __lsx_vldrepl_b(src, 25); src5 = __lsx_vldrepl_b(src, 26); src4 = __lsx_vldrepl_b(src, 27); src3 = __lsx_vldrepl_b(src, 28); src2 = __lsx_vldrepl_b(src, 29); src1 = __lsx_vldrepl_b(src, 30); src0 = __lsx_vldrepl_b(src, 31); LSX_ST_8X16(src0, src1, src2, src3, src4, src5, src6, src7, dst, dst_stride); LSX_ST_8X16(src8, src9, src10, src11, src12, src13, src14, src15, dst, dst_stride); LSX_ST_8X16(src16, src17, src18, src19, src20, src21, src22, src23, dst, dst_stride); LSX_ST_8X16(src24, src25, src26, src27, src28, src29, src30, src31, dst, dst_stride); } void ff_dc_4x4_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left, const uint8_t *src_top) { __m128i tmp0, tmp1, dst0; tmp0 = __lsx_vldrepl_w(src_top, 0); tmp1 = __lsx_vldrepl_w(src_left, 0); dst0 = __lsx_vilvl_w(tmp1, tmp0); dst0 = __lsx_vhaddw_hu_bu(dst0, dst0); dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); dst0 = __lsx_vhaddw_du_wu(dst0, dst0); dst0 = __lsx_vsrari_w(dst0, 3); dst0 = __lsx_vshuf4i_b(dst0, 0); __lsx_vstelm_w(dst0, dst, 0, 0); dst += dst_stride; __lsx_vstelm_w(dst0, dst, 0, 0); dst += dst_stride; __lsx_vstelm_w(dst0, dst, 0, 0); dst += dst_stride; __lsx_vstelm_w(dst0, dst, 0, 0); } #define INTRA_DC_TL_4X4(dir) \ void ff_dc_##dir##_4x4_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ const uint8_t *left, \ const uint8_t *top) \ { \ __m128i tmp0, dst0; \ \ tmp0 = __lsx_vldrepl_w(dir, 0); \ dst0 = __lsx_vhaddw_hu_bu(tmp0, tmp0); \ dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); \ dst0 = __lsx_vsrari_w(dst0, 2); \ dst0 = __lsx_vshuf4i_b(dst0, 0); \ __lsx_vstelm_w(dst0, dst, 0, 0); \ dst += dst_stride; \ __lsx_vstelm_w(dst0, dst, 0, 0); \ dst += dst_stride; \ __lsx_vstelm_w(dst0, dst, 0, 0); \ dst += dst_stride; \ __lsx_vstelm_w(dst0, dst, 0, 0); \ } INTRA_DC_TL_4X4(top); INTRA_DC_TL_4X4(left); void ff_dc_8x8_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left, const uint8_t *src_top) { __m128i tmp0, tmp1, dst0; tmp0 = __lsx_vldrepl_d(src_top, 0); tmp1 = __lsx_vldrepl_d(src_left, 0); dst0 = __lsx_vilvl_d(tmp1, tmp0); dst0 = __lsx_vhaddw_hu_bu(dst0, dst0); dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); dst0 = __lsx_vhaddw_du_wu(dst0, dst0); dst0 = __lsx_vhaddw_qu_du(dst0, dst0); dst0 = __lsx_vsrari_w(dst0, 4); dst0 = __lsx_vreplvei_b(dst0, 0); __lsx_vstelm_d(dst0, dst, 0, 0); dst += dst_stride; __lsx_vstelm_d(dst0, dst, 0, 0); dst += dst_stride; __lsx_vstelm_d(dst0, dst, 0, 0); dst += dst_stride; __lsx_vstelm_d(dst0, dst, 0, 0); dst += dst_stride; __lsx_vstelm_d(dst0, dst, 0, 0); dst += dst_stride; __lsx_vstelm_d(dst0, dst, 0, 0); dst += dst_stride; __lsx_vstelm_d(dst0, dst, 0, 0); dst += dst_stride; __lsx_vstelm_d(dst0, dst, 0, 0); } #define INTRA_DC_TL_8X8(dir) \ void ff_dc_##dir##_8x8_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ const uint8_t *left, \ const uint8_t *top) \ { \ __m128i tmp0, dst0; \ \ tmp0 = __lsx_vldrepl_d(dir, 0); \ dst0 = __lsx_vhaddw_hu_bu(tmp0, tmp0); \ dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); \ dst0 = __lsx_vhaddw_du_wu(dst0, dst0); \ dst0 = __lsx_vsrari_w(dst0, 3); \ dst0 = __lsx_vreplvei_b(dst0, 0); \ __lsx_vstelm_d(dst0, dst, 0, 0); \ dst += dst_stride; \ __lsx_vstelm_d(dst0, dst, 0, 0); \ dst += dst_stride; \ __lsx_vstelm_d(dst0, dst, 0, 0); \ dst += dst_stride; \ __lsx_vstelm_d(dst0, dst, 0, 0); \ dst += dst_stride; \ __lsx_vstelm_d(dst0, dst, 0, 0); \ dst += dst_stride; \ __lsx_vstelm_d(dst0, dst, 0, 0); \ dst += dst_stride; \ __lsx_vstelm_d(dst0, dst, 0, 0); \ dst += dst_stride; \ __lsx_vstelm_d(dst0, dst, 0, 0); \ } INTRA_DC_TL_8X8(top); INTRA_DC_TL_8X8(left); void ff_dc_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left, const uint8_t *src_top) { __m128i tmp0, tmp1, dst0; ptrdiff_t stride2 = dst_stride << 1; ptrdiff_t stride3 = stride2 + dst_stride; ptrdiff_t stride4 = stride2 << 1; tmp0 = __lsx_vld(src_top, 0); tmp1 = __lsx_vld(src_left, 0); DUP2_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp0, tmp1); dst0 = __lsx_vadd_h(tmp0, tmp1); dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); dst0 = __lsx_vhaddw_du_wu(dst0, dst0); dst0 = __lsx_vhaddw_qu_du(dst0, dst0); dst0 = __lsx_vsrari_w(dst0, 5); dst0 = __lsx_vreplvei_b(dst0, 0); LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride, stride2, stride3, stride4); dst += stride4; LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride, stride2, stride3, stride4); } #define INTRA_DC_TL_16X16(dir) \ void ff_dc_##dir##_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ const uint8_t *left, \ const uint8_t *top) \ { \ __m128i tmp0, dst0; \ ptrdiff_t stride2 = dst_stride << 1; \ ptrdiff_t stride3 = stride2 + dst_stride; \ ptrdiff_t stride4 = stride2 << 1; \ \ tmp0 = __lsx_vld(dir, 0); \ dst0 = __lsx_vhaddw_hu_bu(tmp0, tmp0); \ dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); \ dst0 = __lsx_vhaddw_du_wu(dst0, dst0); \ dst0 = __lsx_vhaddw_qu_du(dst0, dst0); \ dst0 = __lsx_vsrari_w(dst0, 4); \ dst0 = __lsx_vreplvei_b(dst0, 0); \ LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, \ dst_stride, stride2, stride3, stride4); \ dst += stride4; \ LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, \ dst_stride, stride2, stride3, stride4); \ } INTRA_DC_TL_16X16(top); INTRA_DC_TL_16X16(left); void ff_dc_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left, const uint8_t *src_top) { __m128i tmp0, tmp1, tmp2, tmp3, dst0; DUP2_ARG2(__lsx_vld, src_top, 0, src_top, 16, tmp0, tmp1); DUP2_ARG2(__lsx_vld, src_left, 0, src_left, 16, tmp2, tmp3); DUP4_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp2, tmp2, tmp3, tmp3, tmp0, tmp1, tmp2, tmp3); DUP2_ARG2(__lsx_vadd_h, tmp0, tmp1, tmp2, tmp3, tmp0, tmp1); dst0 = __lsx_vadd_h(tmp0, tmp1); dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); dst0 = __lsx_vhaddw_du_wu(dst0, dst0); dst0 = __lsx_vhaddw_qu_du(dst0, dst0); dst0 = __lsx_vsrari_w(dst0, 6); dst0 = __lsx_vreplvei_b(dst0, 0); LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride); LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride); LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride); LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, dst_stride); } #define INTRA_DC_TL_32X32(dir) \ void ff_dc_##dir##_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ const uint8_t *left, \ const uint8_t *top) \ { \ __m128i tmp0, tmp1, dst0; \ \ DUP2_ARG2(__lsx_vld, dir, 0, dir, 16, tmp0, tmp1); \ DUP2_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp0, tmp1); \ dst0 = __lsx_vadd_h(tmp0, tmp1); \ dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); \ dst0 = __lsx_vhaddw_du_wu(dst0, dst0); \ dst0 = __lsx_vhaddw_qu_du(dst0, dst0); \ dst0 = __lsx_vsrari_w(dst0, 5); \ dst0 = __lsx_vreplvei_b(dst0, 0); \ LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, \ dst, dst_stride); \ LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, \ dst, dst_stride); \ LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, \ dst, dst_stride); \ LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, \ dst, dst_stride); \ } INTRA_DC_TL_32X32(top); INTRA_DC_TL_32X32(left); #define INTRA_PREDICT_VALDC_16X16_LSX(val) \ void ff_dc_##val##_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ const uint8_t *left, const uint8_t *top) \ { \ __m128i out = __lsx_vldi(val); \ ptrdiff_t stride2 = dst_stride << 1; \ ptrdiff_t stride3 = stride2 + dst_stride; \ ptrdiff_t stride4 = stride2 << 1; \ \ LSX_ST_8(out, out, out, out, out, out, out, out, dst, \ dst_stride, stride2, stride3, stride4); \ dst += stride4; \ LSX_ST_8(out, out, out, out, out, out, out, out, dst, \ dst_stride, stride2, stride3, stride4); \ } INTRA_PREDICT_VALDC_16X16_LSX(127); INTRA_PREDICT_VALDC_16X16_LSX(128); INTRA_PREDICT_VALDC_16X16_LSX(129); #define INTRA_PREDICT_VALDC_32X32_LSX(val) \ void ff_dc_##val##_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ const uint8_t *left, const uint8_t *top) \ { \ __m128i out = __lsx_vldi(val); \ \ LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\ LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\ LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\ LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\ } INTRA_PREDICT_VALDC_32X32_LSX(127); INTRA_PREDICT_VALDC_32X32_LSX(128); INTRA_PREDICT_VALDC_32X32_LSX(129); void ff_tm_4x4_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left, const uint8_t *src_top_ptr) { uint8_t top_left = src_top_ptr[-1]; __m128i tmp0, tmp1, tmp2, tmp3, reg0, reg1; __m128i src0, src1, src2, src3; __m128i dst0, dst1, dst2, dst3; reg0 = __lsx_vreplgr2vr_h(top_left); reg1 = __lsx_vld(src_top_ptr, 0); DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left, 3, tmp3, tmp2, tmp1, tmp0); DUP4_ARG2(__lsx_vilvl_b, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, src0, src1, src2, src3); DUP4_ARG2(__lsx_vhaddw_hu_bu, src0, src0, src1, src1, src2, src2, src3, src3, dst0, dst1, dst2, dst3); DUP4_ARG2(__lsx_vssub_hu, dst0, reg0, dst1, reg0, dst2, reg0, dst3, reg0, dst0, dst1, dst2, dst3); DUP4_ARG2(__lsx_vsat_hu, dst0, 7, dst1, 7, dst2, 7, dst3, 7, dst0, dst1, dst2, dst3); DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst0, dst1); __lsx_vstelm_w(dst0, dst, 0, 0); dst += dst_stride; __lsx_vstelm_w(dst0, dst, 0, 2); dst += dst_stride; __lsx_vstelm_w(dst1, dst, 0, 0); dst += dst_stride; __lsx_vstelm_w(dst1, dst, 0, 2); } void ff_tm_8x8_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left, const uint8_t *src_top_ptr) { uint8_t top_left = src_top_ptr[-1]; __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; __m128i src0, src1, src2, src3, src4, src5, src6, src7; __m128i reg0, reg1; reg0 = __lsx_vreplgr2vr_h(top_left); reg1 = __lsx_vld(src_top_ptr, 0); DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left, 3, tmp7, tmp6, tmp5, tmp4); DUP4_ARG2(__lsx_vldrepl_b, src_left, 4, src_left, 5, src_left, 6, src_left, 7, tmp3, tmp2, tmp1, tmp0); DUP4_ARG2(__lsx_vilvl_b, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, src0, src1, src2, src3); DUP4_ARG2(__lsx_vilvl_b, tmp4, reg1, tmp5, reg1, tmp6, reg1, tmp7, reg1, src4, src5, src6, src7); DUP4_ARG2(__lsx_vhaddw_hu_bu, src0, src0, src1, src1, src2, src2, src3, src3, src0, src1, src2, src3); DUP4_ARG2(__lsx_vhaddw_hu_bu, src4, src4, src5, src5, src6, src6, src7, src7, src4, src5, src6, src7); DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, src0, src1, src2, src3); DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, src4, src5, src6, src7); DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, src0, src1, src2, src3); DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, src4, src5, src6, src7); DUP4_ARG2(__lsx_vpickev_b, src1, src0, src3, src2, src5, src4, src7, src6, src0, src1, src2, src3); __lsx_vstelm_d(src0, dst, 0, 0); dst += dst_stride; __lsx_vstelm_d(src0, dst, 0, 1); dst += dst_stride; __lsx_vstelm_d(src1, dst, 0, 0); dst += dst_stride; __lsx_vstelm_d(src1, dst, 0, 1); dst += dst_stride; __lsx_vstelm_d(src2, dst, 0, 0); dst += dst_stride; __lsx_vstelm_d(src2, dst, 0, 1); dst += dst_stride; __lsx_vstelm_d(src3, dst, 0, 0); dst += dst_stride; __lsx_vstelm_d(src3, dst, 0, 1); } void ff_tm_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left, const uint8_t *src_top_ptr) { uint8_t top_left = src_top_ptr[-1]; __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; __m128i tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15; __m128i src0, src1, src2, src3, src4, src5, src6, src7; __m128i reg0, reg1; ptrdiff_t stride2 = dst_stride << 1; ptrdiff_t stride3 = stride2 + dst_stride; ptrdiff_t stride4 = stride2 << 1; reg0 = __lsx_vreplgr2vr_h(top_left); reg1 = __lsx_vld(src_top_ptr, 0); DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left, 3, tmp15, tmp14, tmp13, tmp12); DUP4_ARG2(__lsx_vldrepl_b, src_left, 4, src_left, 5, src_left, 6, src_left, 7, tmp11, tmp10, tmp9, tmp8); DUP4_ARG2(__lsx_vldrepl_b, src_left, 8, src_left, 9, src_left, 10, src_left, 11, tmp7, tmp6, tmp5, tmp4); DUP4_ARG2(__lsx_vldrepl_b, src_left, 12, src_left, 13, src_left, 14, src_left, 15, tmp3, tmp2, tmp1, tmp0); DUP4_ARG2(__lsx_vaddwev_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, src0, src1, src2, src3); DUP4_ARG2(__lsx_vaddwod_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, src4, src5, src6, src7); DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, src0, src1, src2, src3); DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, src4, src5, src6, src7); DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, src0, src1, src2, src3); DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, src4, src5, src6, src7); DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, tmp0, tmp1, tmp2, tmp3); DUP4_ARG2(__lsx_vaddwev_h_bu, tmp4, reg1, tmp5, reg1, tmp6, reg1, tmp7, reg1, src0, src1, src2, src3); DUP4_ARG2(__lsx_vaddwod_h_bu, tmp4, reg1, tmp5, reg1, tmp6, reg1, tmp7, reg1, src4, src5, src6, src7); DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, src0, src1, src2, src3); DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, src4, src5, src6, src7); DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, src0, src1, src2, src3); DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, src4, src5, src6, src7); DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, tmp4, tmp5, tmp6, tmp7); DUP4_ARG2(__lsx_vaddwev_h_bu, tmp8, reg1, tmp9, reg1, tmp10, reg1, tmp11, reg1, src0, src1, src2, src3); DUP4_ARG2(__lsx_vaddwod_h_bu, tmp8, reg1, tmp9, reg1, tmp10, reg1, tmp11, reg1, src4, src5, src6, src7); DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, src0, src1, src2, src3); DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, src4, src5, src6, src7); DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, src0, src1, src2, src3); DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, src4, src5, src6, src7); DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, tmp8, tmp9, tmp10, tmp11); DUP4_ARG2(__lsx_vaddwev_h_bu, tmp12, reg1, tmp13, reg1, tmp14, reg1, tmp15, reg1, src0, src1, src2, src3); DUP4_ARG2(__lsx_vaddwod_h_bu, tmp12, reg1, tmp13, reg1, tmp14, reg1, tmp15, reg1, src4, src5, src6, src7); DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, src0, src1, src2, src3); DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, src4, src5, src6, src7); DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, src0, src1, src2, src3); DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, src4, src5, src6, src7); DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, tmp12, tmp13, tmp14, tmp15); LSX_ST_8(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, dst, dst_stride, stride2, stride3, stride4); dst += stride4; LSX_ST_8(tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15, dst, dst_stride, stride2, stride3, stride4); } void ff_tm_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left, const uint8_t *src_top_ptr) { uint8_t top_left = src_top_ptr[-1]; uint32_t loop_cnt; __m128i tmp0, tmp1, tmp2, tmp3, reg0, reg1, reg2; __m128i src0, src1, src2, src3, src4, src5, src6, src7; __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; reg0 = __lsx_vreplgr2vr_h(top_left); DUP2_ARG2(__lsx_vld, src_top_ptr, 0, src_top_ptr, 16, reg1, reg2); src_left += 28; for (loop_cnt = 8; loop_cnt--;) { DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left, 3, tmp3, tmp2, tmp1, tmp0); src_left -= 4; DUP4_ARG2(__lsx_vaddwev_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, src0, src1, src2, src3); DUP4_ARG2(__lsx_vaddwod_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, src4, src5, src6, src7); DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, src0, src1, src2, src3); DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, src4, src5, src6, src7); DUP4_ARG2(__lsx_vaddwev_h_bu, tmp0, reg2, tmp1, reg2, tmp2, reg2, tmp3, reg2, dst0, dst1, dst2, dst3); DUP4_ARG2(__lsx_vaddwod_h_bu, tmp0, reg2, tmp1, reg2, tmp2, reg2, tmp3, reg2, dst4, dst5, dst6, dst7); DUP4_ARG2(__lsx_vssub_hu, dst0, reg0, dst1, reg0, dst2, reg0, dst3, reg0, dst0, dst1, dst2, dst3); DUP4_ARG2(__lsx_vssub_hu, dst4, reg0, dst5, reg0, dst6, reg0, dst7, reg0, dst4, dst5, dst6, dst7); DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, src0, src1, src2, src3); DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, src4, src5, src6, src7); DUP4_ARG2(__lsx_vsat_hu, dst0, 7, dst1, 7, dst2, 7, dst3, 7, dst0, dst1, dst2, dst3); DUP4_ARG2(__lsx_vsat_hu, dst4, 7, dst5, 7, dst6, 7, dst7, 7, dst4, dst5, dst6, dst7); DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, src0, src1, src2, src3); DUP4_ARG2(__lsx_vpackev_b, dst4, dst0, dst5, dst1, dst6, dst2, dst7, dst3, dst0, dst1, dst2, dst3); __lsx_vst(src0, dst, 0); __lsx_vst(dst0, dst, 16); dst += dst_stride; __lsx_vst(src1, dst, 0); __lsx_vst(dst1, dst, 16); dst += dst_stride; __lsx_vst(src2, dst, 0); __lsx_vst(dst2, dst, 16); dst += dst_stride; __lsx_vst(src3, dst, 0); __lsx_vst(dst3, dst, 16); dst += dst_stride; } }