h264: aarch64: intra prediction optimisations

3 files changed, 456 insertions, 0 deletions

diff --git a/libavcodec/aarch64/Makefile b/libavcodec/aarch64/Makefile
index 2afff297dc..d001b345c0 100644
--- a/libavcodec/aarch64/Makefile
+++ b/libavcodec/aarch64/Makefile
@@ -1,6 +1,7 @@
 OBJS-$(CONFIG_FFT)                      += aarch64/fft_init_aarch64.o
 OBJS-$(CONFIG_H264CHROMA)               += aarch64/h264chroma_init_aarch64.o
 OBJS-$(CONFIG_H264DSP)                  += aarch64/h264dsp_init_aarch64.o
+OBJS-$(CONFIG_H264PRED)                 += aarch64/h264pred_init.o
 OBJS-$(CONFIG_H264QPEL)                 += aarch64/h264qpel_init_aarch64.o
 OBJS-$(CONFIG_HPELDSP)                  += aarch64/hpeldsp_init_aarch64.o
 OBJS-$(CONFIG_IMDCT15)                  += aarch64/imdct15_init.o
@@ -18,6 +19,7 @@ NEON-OBJS-$(CONFIG_FFT)                 += aarch64/fft_neon.o
 NEON-OBJS-$(CONFIG_H264CHROMA)          += aarch64/h264cmc_neon.o
 NEON-OBJS-$(CONFIG_H264DSP)             += aarch64/h264dsp_neon.o              \
                                            aarch64/h264idct_neon.o
+NEON-OBJS-$(CONFIG_H264PRED)            += aarch64/h264pred_neon.o
 NEON-OBJS-$(CONFIG_H264QPEL)            += aarch64/h264qpel_neon.o             \
                                            aarch64/hpeldsp_neon.o
 NEON-OBJS-$(CONFIG_HPELDSP)             += aarch64/hpeldsp_neon.o
diff --git a/libavcodec/aarch64/h264pred_init.c b/libavcodec/aarch64/h264pred_init.c
new file mode 100644
index 0000000000..8f912cbca9
--- /dev/null
+++ b/libavcodec/aarch64/h264pred_init.c
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2009 Mans Rullgard <mans@mansr.com>
+ *
+ * This file is part of Libav.
+ *
+ * Libav is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * Libav is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Libav; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdint.h>
+
+#include "libavutil/attributes.h"
+#include "libavutil/aarch64/cpu.h"
+#include "libavcodec/avcodec.h"
+#include "libavcodec/h264pred.h"
+
+void ff_pred16x16_vert_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred16x16_hor_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred16x16_plane_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred16x16_dc_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred16x16_128_dc_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred16x16_left_dc_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred16x16_top_dc_neon(uint8_t *src, ptrdiff_t stride);
+
+void ff_pred8x8_vert_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred8x8_hor_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred8x8_plane_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred8x8_dc_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred8x8_128_dc_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred8x8_left_dc_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred8x8_top_dc_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred8x8_l0t_dc_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred8x8_0lt_dc_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred8x8_l00_dc_neon(uint8_t *src, ptrdiff_t stride);
+void ff_pred8x8_0l0_dc_neon(uint8_t *src, ptrdiff_t stride);
+
+static av_cold void h264_pred_init_neon(H264PredContext *h, int codec_id,
+                                        const int bit_depth,
+                                        const int chroma_format_idc)
+{
+    const int high_depth = bit_depth > 8;
+
+    if (high_depth)
+        return;
+
+    if (chroma_format_idc <= 1) {
+        h->pred8x8[VERT_PRED8x8     ] = ff_pred8x8_vert_neon;
+        h->pred8x8[HOR_PRED8x8      ] = ff_pred8x8_hor_neon;
+        if (codec_id != AV_CODEC_ID_VP7 && codec_id != AV_CODEC_ID_VP8)
+            h->pred8x8[PLANE_PRED8x8] = ff_pred8x8_plane_neon;
+        h->pred8x8[DC_128_PRED8x8   ] = ff_pred8x8_128_dc_neon;
+        if (codec_id != AV_CODEC_ID_RV40 && codec_id != AV_CODEC_ID_VP7 &&
+            codec_id != AV_CODEC_ID_VP8) {
+            h->pred8x8[DC_PRED8x8     ] = ff_pred8x8_dc_neon;
+            h->pred8x8[LEFT_DC_PRED8x8] = ff_pred8x8_left_dc_neon;
+            h->pred8x8[TOP_DC_PRED8x8 ] = ff_pred8x8_top_dc_neon;
+            h->pred8x8[ALZHEIMER_DC_L0T_PRED8x8] = ff_pred8x8_l0t_dc_neon;
+            h->pred8x8[ALZHEIMER_DC_0LT_PRED8x8] = ff_pred8x8_0lt_dc_neon;
+            h->pred8x8[ALZHEIMER_DC_L00_PRED8x8] = ff_pred8x8_l00_dc_neon;
+            h->pred8x8[ALZHEIMER_DC_0L0_PRED8x8] = ff_pred8x8_0l0_dc_neon;
+        }
+    }
+
+    h->pred16x16[DC_PRED8x8     ] = ff_pred16x16_dc_neon;
+    h->pred16x16[VERT_PRED8x8   ] = ff_pred16x16_vert_neon;
+    h->pred16x16[HOR_PRED8x8    ] = ff_pred16x16_hor_neon;
+    h->pred16x16[LEFT_DC_PRED8x8] = ff_pred16x16_left_dc_neon;
+    h->pred16x16[TOP_DC_PRED8x8 ] = ff_pred16x16_top_dc_neon;
+    h->pred16x16[DC_128_PRED8x8 ] = ff_pred16x16_128_dc_neon;
+    if (codec_id != AV_CODEC_ID_SVQ3 && codec_id != AV_CODEC_ID_RV40 &&
+        codec_id != AV_CODEC_ID_VP7 && codec_id != AV_CODEC_ID_VP8)
+        h->pred16x16[PLANE_PRED8x8  ] = ff_pred16x16_plane_neon;
+}
+
+av_cold void ff_h264_pred_init_aarch64(H264PredContext *h, int codec_id,
+                                       int bit_depth, const int chroma_format_idc)
+{
+    int cpu_flags = av_get_cpu_flags();
+
+    if (have_neon(cpu_flags))
+        h264_pred_init_neon(h, codec_id, bit_depth, chroma_format_idc);
+}
diff --git a/libavcodec/aarch64/h264pred_neon.S b/libavcodec/aarch64/h264pred_neon.S
new file mode 100644
index 0000000000..a38a27f186
--- /dev/null
+++ b/libavcodec/aarch64/h264pred_neon.S
@@ -0,0 +1,361 @@
+/*
+ * Copyright (c) 2009 Mans Rullgard <mans@mansr.com>
+ *
+ * This file is part of Libav.
+ *
+ * Libav is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * Libav is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Libav; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/aarch64/asm.S"
+
+.macro ldcol.8  rd,  rs,  rt,  n=8,  hi=0
+.if \n >= 8 || \hi == 0
+        ld1             {\rd\().b}[0],  [\rs], \rt
+        ld1             {\rd\().b}[1],  [\rs], \rt
+        ld1             {\rd\().b}[2],  [\rs], \rt
+        ld1             {\rd\().b}[3],  [\rs], \rt
+.endif
+.if \n >= 8 || \hi == 1
+        ld1             {\rd\().b}[4],  [\rs], \rt
+        ld1             {\rd\().b}[5],  [\rs], \rt
+        ld1             {\rd\().b}[6],  [\rs], \rt
+        ld1             {\rd\().b}[7],  [\rs], \rt
+.endif
+.if \n == 16
+        ld1             {\rd\().b}[8],  [\rs], \rt
+        ld1             {\rd\().b}[9],  [\rs], \rt
+        ld1             {\rd\().b}[10], [\rs], \rt
+        ld1             {\rd\().b}[11], [\rs], \rt
+        ld1             {\rd\().b}[12], [\rs], \rt
+        ld1             {\rd\().b}[13], [\rs], \rt
+        ld1             {\rd\().b}[14], [\rs], \rt
+        ld1             {\rd\().b}[15], [\rs], \rt
+.endif
+.endm
+
+function ff_pred16x16_128_dc_neon, export=1
+        movi            v0.16b,  #128
+        b               .L_pred16x16_dc_end
+endfunc
+
+function ff_pred16x16_top_dc_neon, export=1
+        sub             x2,  x0,  x1
+        ld1             {v0.16b},  [x2]
+        uaddlv          h0,  v0.16b
+        rshrn           v0.8b,  v0.8h,  #4
+        dup             v0.16b, v0.b[0]
+        b               .L_pred16x16_dc_end
+endfunc
+
+function ff_pred16x16_left_dc_neon, export=1
+        sub             x2,  x0,  #1
+        ldcol.8         v0,  x2,  x1, 16
+        uaddlv          h0,  v0.16b
+        rshrn           v0.8b,  v0.8h,  #4
+        dup             v0.16b, v0.b[0]
+        b               .L_pred16x16_dc_end
+endfunc
+
+function ff_pred16x16_dc_neon, export=1
+        sub             x2,  x0,  x1
+        sub             x3,  x0,  #1
+        ld1             {v0.16b}, [x2]
+        ldcol.8         v1,  x3,  x1, 16
+        uaddlv          h0,  v0.16b
+        uaddlv          h1,  v1.16b
+        add             v0.4h,  v0.4h,  v1.4h
+        rshrn           v0.8b,  v0.8h,  #5
+        dup             v0.16b, v0.b[0]
+.L_pred16x16_dc_end:
+        mov             w3,  #8
+6:      st1             {v0.16b}, [x0], x1
+        st1             {v0.16b}, [x0], x1
+        subs            w3,  w3,  #1
+        b.ne            6b
+        ret
+endfunc
+
+function ff_pred16x16_hor_neon, export=1
+        sub             x2,  x0,  #1
+        mov             w3,  #16
+1:      ld1r            {v0.16b}, [x2], x1
+        st1             {v0.16b}, [x0], x1
+        subs            w3,  w3,  #1
+        b.ne            1b
+        ret
+endfunc
+
+function ff_pred16x16_vert_neon, export=1
+        sub             x2,  x0,  x1
+        add             x1,  x1,  x1
+        ld1             {v0.16b}, [x2], x1
+        mov             w3,  #8
+1:      st1             {v0.16b}, [x0], x1
+        st1             {v0.16b}, [x2], x1
+        subs            w3,  w3,  #1
+        b.ne            1b
+        ret
+endfunc
+
+function ff_pred16x16_plane_neon, export=1
+        sub             x3,  x0,  x1
+        movrel          x4,  p16weight
+        add             x2,  x3,  #8
+        sub             x3,  x3,  #1
+        ld1             {v0.8b},  [x3]
+        ld1             {v2.8b},  [x2], x1
+        ldcol.8         v1,  x3,  x1
+        add             x3,  x3,  x1
+        ldcol.8         v3,  x3,  x1
+        rev64           v0.8b,  v0.8b
+        rev64           v1.8b,  v1.8b
+        uaddl           v7.8h,  v2.8b,  v3.8b
+        usubl           v2.8h,  v2.8b,  v0.8b
+        usubl           v3.8h,  v3.8b,  v1.8b
+        ld1             {v0.8h},     [x4]
+        mul             v2.8h,  v2.8h,  v0.8h
+        mul             v3.8h,  v3.8h,  v0.8h
+        addp            v2.8h,  v2.8h,  v3.8h
+        addp            v2.8h,  v2.8h,  v2.8h
+        addp            v2.4h,  v2.4h,  v2.4h
+        sshll           v3.4s,  v2.4h,  #2
+        saddw           v2.4s,  v3.4s,  v2.4h
+        rshrn           v4.4h,  v2.4s,  #6
+        trn2            v5.4h,  v4.4h,  v4.4h
+        add             v2.4h,  v4.4h,  v5.4h
+        shl             v3.4h,  v2.4h,  #3
+        ext             v7.16b, v7.16b, v7.16b, #14
+        sub             v3.4h,  v3.4h,  v2.4h   // 7 * (b + c)
+        add             v7.4h,  v7.4h,  v0.4h
+        shl             v2.4h,  v7.4h,  #4
+        sub             v2.4h,  v2.4h,  v3.4h
+        shl             v3.4h,  v4.4h,  #4
+        ext             v0.16b, v0.16b, v0.16b, #14
+        sub             v6.4h,  v5.4h,  v3.4h
+        mov             v0.h[0],  wzr
+        mul             v0.8h,  v0.8h,  v4.h[0]
+        dup             v1.8h,  v2.h[0]
+        dup             v2.8h,  v4.h[0]
+        dup             v3.8h,  v6.h[0]
+        shl             v2.8h,  v2.8h,  #3
+        add             v1.8h,  v1.8h,  v0.8h
+        add             v3.8h,  v3.8h,  v2.8h
+        mov             w3,  #16
+1:
+        sqshrun         v0.8b,  v1.8h,  #5
+        add             v1.8h,  v1.8h,  v2.8h
+        sqshrun2        v0.16b, v1.8h,  #5
+        add             v1.8h,  v1.8h,  v3.8h
+        st1             {v0.16b}, [x0], x1
+        subs            w3,  w3,  #1
+        b.ne            1b
+        ret
+endfunc
+
+const   p16weight, align=4
+        .short          1,2,3,4,5,6,7,8
+endconst
+const   p8weight, align=4
+        .short          1,2,3,4,1,2,3,4
+endconst
+
+function ff_pred8x8_hor_neon, export=1
+        sub             x2,  x0,  #1
+        mov             w3,  #8
+1:      ld1r            {v0.8b},  [x2], x1
+        st1             {v0.8b},  [x0], x1
+        subs            w3,  w3,  #1
+        b.ne            1b
+        ret
+endfunc
+
+function ff_pred8x8_vert_neon, export=1
+        sub             x2,  x0,  x1
+        lsl             x1,  x1,  #1
+        ld1             {v0.8b},  [x2], x1
+        mov             w3,  #4
+1:      st1             {v0.8b},  [x0], x1
+        st1             {v0.8b},  [x2], x1
+        subs            w3,  w3,  #1
+        b.ne            1b
+        ret
+endfunc
+
+function ff_pred8x8_plane_neon, export=1
+        sub             x3,  x0,  x1
+        movrel          x4,  p8weight
+        movrel          x5,  p16weight
+        add             x2,  x3,  #4
+        sub             x3,  x3,  #1
+        ld1             {v0.s}[0],  [x3]
+        ld1             {v2.s}[0],  [x2], x1
+        ldcol.8         v0,  x3,  x1,  4,  hi=1
+        add             x3,  x3,  x1
+        ldcol.8         v3,  x3,  x1,  4
+        uaddl           v7.8h,  v2.8b,  v3.8b
+        rev32           v0.8b,  v0.8b
+        trn1            v2.2s,  v2.2s,  v3.2s
+        usubl           v2.8h,  v2.8b,  v0.8b
+        ld1             {v6.8h},  [x4]
+        mul             v2.8h,  v2.8h,  v6.8h
+        ld1             {v0.8h},  [x5]
+        saddlp          v2.4s,  v2.8h
+        addp            v2.4s,  v2.4s,  v2.4s
+        shl             v3.4s,  v2.4s,  #4
+        add             v2.4s,  v3.4s,  v2.4s
+        rshrn           v5.4h,  v2.4s,  #5
+        addp            v2.4h,  v5.4h,  v5.4h
+        shl             v3.4h,  v2.4h,  #1
+        add             v3.4h,  v3.4h,  v2.4h
+        rev64           v7.4h,  v7.4h
+        add             v7.4h,  v7.4h,  v0.4h
+        shl             v2.4h,  v7.4h,  #4
+        sub             v2.4h,  v2.4h,  v3.4h
+        ext             v0.16b, v0.16b, v0.16b, #14
+        mov             v0.h[0],  wzr
+        mul             v0.8h,  v0.8h,  v5.h[0]
+        dup             v1.8h,  v2.h[0]
+        dup             v2.8h,  v5.h[1]
+        add             v1.8h,  v1.8h,  v0.8h
+        mov             w3,  #8
+1:
+        sqshrun         v0.8b,  v1.8h,  #5
+        add             v1.8h,  v1.8h,  v2.8h
+        st1             {v0.8b},  [x0], x1
+        subs            w3,  w3,  #1
+        b.ne            1b
+        ret
+endfunc
+
+function ff_pred8x8_128_dc_neon, export=1
+        movi            v0.8b,  #128
+        movi            v1.8b,  #128
+        b               .L_pred8x8_dc_end
+endfunc
+
+function ff_pred8x8_top_dc_neon, export=1
+        sub             x2,  x0,  x1
+        ld1             {v0.8b},  [x2]
+        uaddlp          v0.4h,  v0.8b
+        addp            v0.4h,  v0.4h,  v0.4h
+        zip1            v0.8h,  v0.8h,  v0.8h
+        rshrn           v2.8b,  v0.8h,  #2
+        zip1            v0.8b,  v2.8b,  v2.8b
+        zip1            v1.8b,  v2.8b,  v2.8b
+        b               .L_pred8x8_dc_end
+endfunc
+
+function ff_pred8x8_left_dc_neon, export=1
+        sub             x2,  x0,  #1
+        ldcol.8         v0,  x2,  x1
+        uaddlp          v0.4h,  v0.8b
+        addp            v0.4h,  v0.4h,  v0.4h
+        rshrn           v2.8b,  v0.8h,  #2
+        dup             v1.8b,  v2.b[1]
+        dup             v0.8b,  v2.b[0]
+        b               .L_pred8x8_dc_end
+endfunc
+
+function ff_pred8x8_dc_neon, export=1
+        sub             x2,  x0,  x1
+        sub             x3,  x0,  #1
+        ld1             {v0.8b}, [x2]
+        ldcol.8         v1,  x3,  x1
+        uaddlp          v0.4h,  v0.8b
+        uaddlp          v1.4h,  v1.8b
+        trn1            v2.2s,  v0.2s,  v1.2s
+        trn2            v3.2s,  v0.2s,  v1.2s
+        addp            v4.4h,  v2.4h,  v3.4h
+        addp            v5.4h,  v4.4h,  v4.4h
+        rshrn           v6.8b,  v5.8h,  #3
+        rshrn           v7.8b,  v4.8h,  #2
+        dup             v0.8b,  v6.b[0]
+        dup             v2.8b,  v7.b[2]
+        dup             v1.8b,  v7.b[3]
+        dup             v3.8b,  v6.b[1]
+        zip1            v0.2s,  v0.2s,  v2.2s
+        zip1            v1.2s,  v1.2s,  v3.2s
+.L_pred8x8_dc_end:
+        mov             w3,  #4
+        add             x2,  x0,  x1,  lsl #2
+6:      st1             {v0.8b},  [x0], x1
+        st1             {v1.8b},  [x2], x1
+        subs            w3,  w3,  #1
+        b.ne            6b
+        ret
+endfunc
+
+function ff_pred8x8_l0t_dc_neon, export=1
+        sub             x2,  x0,  x1
+        sub             x3,  x0,  #1
+        ld1             {v0.8b},  [x2]
+        ldcol.8         v1,  x3,  x1,  4
+        zip1            v0.4s,  v0.4s,  v1.4s
+        uaddlp          v0.8h,  v0.16b
+        addp            v0.8h,  v0.8h,  v0.8h
+        addp            v1.4h,  v0.4h,  v0.4h
+        rshrn           v2.8b,  v0.8h,  #2
+        rshrn           v3.8b,  v1.8h,  #3
+        dup             v4.8b,  v3.b[0]
+        dup             v6.8b,  v2.b[2]
+        dup             v5.8b,  v2.b[0]
+        zip1            v0.2s,  v4.2s,  v6.2s
+        zip1            v1.2s,  v5.2s,  v6.2s
+        b               .L_pred8x8_dc_end
+endfunc
+
+function ff_pred8x8_l00_dc_neon, export=1
+        sub             x2,  x0,  #1
+        ldcol.8         v0,  x2,  x1,  4
+        uaddlp          v0.4h,  v0.8b
+        addp            v0.4h,  v0.4h,  v0.4h
+        rshrn           v0.8b,  v0.8h,  #2
+        movi            v1.8b,  #128
+        dup             v0.8b,  v0.b[0]
+        b               .L_pred8x8_dc_end
+endfunc
+
+function ff_pred8x8_0lt_dc_neon, export=1
+        add             x3,  x0,  x1,  lsl #2
+        sub             x2,  x0,  x1
+        sub             x3,  x3,  #1
+        ld1             {v0.8b},  [x2]
+        ldcol.8         v1,  x3,  x1,  4,  hi=1
+        zip1            v0.4s,  v0.4s,  v1.4s
+        uaddlp          v0.8h,  v0.16b
+        addp            v0.8h,  v0.8h,  v0.8h
+        addp            v1.4h,  v0.4h,  v0.4h
+        rshrn           v2.8b,  v0.8h,  #2
+        rshrn           v3.8b,  v1.8h,  #3
+        dup             v4.8b,  v2.b[0]
+        dup             v5.8b,  v2.b[3]
+        dup             v6.8b,  v2.b[2]
+        dup             v7.8b,  v3.b[1]
+        zip1            v0.2s,  v4.2s,  v6.2s
+        zip1            v1.2s,  v5.2s,  v7.2s
+        b               .L_pred8x8_dc_end
+endfunc
+
+function ff_pred8x8_0l0_dc_neon, export=1
+        add             x2,  x0,  x1,  lsl #2
+        sub             x2,  x2,  #1
+        ldcol.8         v1,  x2,  x1,  4
+        uaddlp          v2.4h,  v1.8b
+        addp            v2.4h,  v2.4h,  v2.4h
+        rshrn           v1.8b,  v2.8h,  #2
+        movi            v0.8b,  #128
+        dup             v1.8b,  v1.b[0]
+        b               .L_pred8x8_dc_end
+endfunc