H.264 deblocking optimizations (mmx for chroma_bS4 case, convert existing cases to 8-bit math)

Originally committed as revision 4271 to svn://svn.ffmpeg.org/ffmpeg/trunk

1 files changed, 235 insertions, 171 deletions

diff --git a/libavcodec/i386/dsputil_mmx.c b/libavcodec/i386/dsputil_mmx.c
index 263e50af28..b767f9d8e3 100644
--- a/libavcodec/i386/dsputil_mmx.c
+++ b/libavcodec/i386/dsputil_mmx.c
@@ -45,6 +45,7 @@ static const uint64_t ff_pw_16 attribute_used __attribute__ ((aligned(8))) = 0x0
 static const uint64_t ff_pw_32 attribute_used __attribute__ ((aligned(8))) = 0x0020002000200020ULL;
 static const uint64_t ff_pw_15 attribute_used __attribute__ ((aligned(8))) = 0x000F000F000F000FULL;
 
+static const uint64_t ff_pb_3F attribute_used __attribute__ ((aligned(8))) = 0x3F3F3F3F3F3F3F3FULL;
 static const uint64_t ff_pb_FC attribute_used __attribute__ ((aligned(8))) = 0xFCFCFCFCFCFCFCFCULL;
 
 #define JUMPALIGN() __asm __volatile (".balign 8"::)
@@ -692,204 +693,265 @@ static void h263_h_loop_filter_mmx(uint8_t *src, int stride, int qscale){
     );
 }
 
-// dst = ABS( a - b )
-#define MMABS_DIFF_MMX2(a,b,dst,z)\
-    "movq    " #b ", " #dst " \n\t"\
-    "movq    " #a ", " #z   " \n\t"\
-    "psubusw " #b ", " #z   " \n\t"\
-    "psubusw " #a ", " #dst " \n\t"\
-    "pmaxsw  " #z ", " #dst " \n\t"
-
-// a = clip( a, -tc, tc )
-#define CLIP_MMX2(a,tc,z)\
-    "pxor    " #z  ", " #z "  \n\t"\
-    "psubw   " #tc ", " #z "  \n\t"\
-    "pmaxsw  " #z  ", " #a "  \n\t"\
-    "pminsw  " #tc ", " #a "  \n\t"
-
-// in: mm0=p1, mm1=p0, mm2=q0, mm3=q1
-// out: mm7 = do we filter this pixel?
-#define H264_DEBLOCK_THRESH(alpha,beta)\
-    "pxor      %%mm7, %%mm7     \n\t"\
-    "punpcklbw %%mm7, %%mm0     \n\t"\
-    "punpcklbw %%mm7, %%mm1     \n\t"\
-    "punpcklbw %%mm7, %%mm2     \n\t"\
-    "punpcklbw %%mm7, %%mm3     \n\t"\
-    MMABS_DIFF_MMX2(%%mm1, %%mm2, %%mm5, %%mm4)\
-    "movd " #alpha ", %%mm6     \n\t"\
-    "pshufw    $0, %%mm6, %%mm6 \n\t"\
-    "pcmpgtw   %%mm5, %%mm6     \n\t" /* ABS(p0-q0) < alpha */\
-    MMABS_DIFF_MMX2(%%mm0, %%mm1, %%mm5, %%mm4)\
-    MMABS_DIFF_MMX2(%%mm3, %%mm2, %%mm7, %%mm4)\
-    "pmaxsw    %%mm7, %%mm5     \n\t"\
-    "movd  " #beta ", %%mm7     \n\t"\
-    "pshufw    $0, %%mm7, %%mm7 \n\t"\
-    "movq      %%mm7, %%mm4     \n\t"\
-    "pcmpgtw   %%mm5, %%mm7     \n\t" /* ABS(p1-p0) < beta && ABS(q1-q0) < beta */\
-    "pand      %%mm6, %%mm7     \n\t"
-
-// in: mm0=p1, mm1=p0, mm2=q0, mm3=q1, mm6=tc
-// out: mm1=p0', mm2=q0'
-#define H264_DEBLOCK_P0_Q0(pw4)\
-    "movq   " #pw4 ", %%mm4     \n\t"\
-    "movq      %%mm2, %%mm5     \n\t"\
-    "paddw     %%mm4, %%mm0     \n\t"\
-    "psubw     %%mm1, %%mm5     \n\t"\
-    "psubw     %%mm3, %%mm0     \n\t"\
-    "psllw     $2,    %%mm5     \n\t"\
-    "paddw     %%mm0, %%mm5     \n\t"\
-    "psraw     $3,    %%mm5     \n\t" /* mm5 = (((q0 - p0) << 2) + (p1 - q1) + 4) >> 3 */\
-    CLIP_MMX2(%%mm5, %%mm6, %%mm4)    /* delta = clip( mm5, -tc, tc ) */\
-    "paddw     %%mm5, %%mm1     \n\t" /* p0 += delta */\
-    "psubw     %%mm5, %%mm2     \n\t" /* q0 -= delta */
-
-// in: mm1=p0, mm2=q0, mm6=tc0
-// out: mm5=delta
-#define H264_DEBLOCK_DELTA_PQ1(p1,p2,z)\
-    "movq      %%mm1, %%mm5     \n\t"\
-    "pavgb     %%mm2, %%mm5     \n\t"\
-    "paddw   " #p2 ", %%mm5     \n\t"\
-    "psraw     $1, %%mm5        \n\t"\
-    "psubw   " #p1 ", %%mm5     \n\t" /* ( ( q2 + ((p0+q0+1)>>1) ) >> 1 ) - q1 */\
-    CLIP_MMX2(%%mm5, %%mm6, z)
-
-static inline void h264_loop_filter_luma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int tc0)
+
+// out: o = |x-y|>a
+// clobbers: t
+#define DIFF_GT_MMX(x,y,a,o,t)\
+    "movq     "#y", "#t"  \n\t"\
+    "movq     "#x", "#o"  \n\t"\
+    "psubusb  "#x", "#t"  \n\t"\
+    "psubusb  "#y", "#o"  \n\t"\
+    "por      "#t", "#o"  \n\t"\
+    "psubusb  "#a", "#o"  \n\t"
+
+// in: mm0=p1 mm1=p0 mm2=q0 mm3=q1
+// out: mm5=beta-1, mm7=mask
+// clobbers: mm4,mm6
+#define H264_DEBLOCK_MASK(alpha1, beta1) \
+    "pshufw $0, "#alpha1", %%mm4 \n\t"\
+    "pshufw $0, "#beta1 ", %%mm5 \n\t"\
+    "packuswb  %%mm4, %%mm4      \n\t"\
+    "packuswb  %%mm5, %%mm5      \n\t"\
+    DIFF_GT_MMX(%%mm1, %%mm2, %%mm4, %%mm7, %%mm6) /* |p0-q0| > alpha-1 */\
+    DIFF_GT_MMX(%%mm0, %%mm1, %%mm5, %%mm4, %%mm6) /* |p1-p0| > beta-1 */\
+    "por       %%mm4, %%mm7      \n\t"\
+    DIFF_GT_MMX(%%mm3, %%mm2, %%mm5, %%mm4, %%mm6) /* |q1-q0| > beta-1 */\
+    "por       %%mm4, %%mm7      \n\t"\
+    "pxor      %%mm6, %%mm6      \n\t"\
+    "pcmpeqb   %%mm6, %%mm7      \n\t"
+
+// in: mm0=p1 mm1=p0 mm2=q0 mm3=q1 mm7=(tc&mask)
+// out: mm1=p0' mm2=q0'
+// clobbers: mm0,3-6
+#define H264_DEBLOCK_P0_Q0(pb_01, pb_3f)\
+        /* a = q0^p0^((p1-q1)>>2) */\
+        "movq    %%mm0, %%mm4  \n\t"\
+        "psubb   %%mm3, %%mm4  \n\t"\
+        "psrlw   $2,    %%mm4  \n\t"\
+        "pxor    %%mm1, %%mm4  \n\t"\
+        "pxor    %%mm2, %%mm4  \n\t"\
+        /* b = p0^(q1>>2) */\
+        "psrlw   $2,    %%mm3  \n\t"\
+        "pand "#pb_3f", %%mm3  \n\t"\
+        "movq    %%mm1, %%mm5  \n\t"\
+        "pxor    %%mm3, %%mm5  \n\t"\
+        /* c = q0^(p1>>2) */\
+        "psrlw   $2,    %%mm0  \n\t"\
+        "pand "#pb_3f", %%mm0  \n\t"\
+        "movq    %%mm2, %%mm6  \n\t"\
+        "pxor    %%mm0, %%mm6  \n\t"\
+        /* d = (c^b) & ~(b^a) & 1 */\
+        "pxor    %%mm5, %%mm6  \n\t"\
+        "pxor    %%mm4, %%mm5  \n\t"\
+        "pandn   %%mm6, %%mm5  \n\t"\
+        "pand "#pb_01", %%mm5  \n\t"\
+        /* delta = (avg(q0, p1>>2) + (d&a))
+         *       - (avg(p0, q1>>2) + (d&~a)) */\
+        "pavgb   %%mm2, %%mm0  \n\t"\
+        "movq    %%mm5, %%mm6  \n\t"\
+        "pand    %%mm4, %%mm6  \n\t"\
+        "paddusb %%mm6, %%mm0  \n\t"\
+        "pavgb   %%mm1, %%mm3  \n\t"\
+        "pandn   %%mm5, %%mm4  \n\t"\
+        "paddusb %%mm4, %%mm3  \n\t"\
+        /* p0 += clip(delta, -tc0, tc0)
+         * q0 -= clip(delta, -tc0, tc0) */\
+        "movq    %%mm0, %%mm4  \n\t"\
+        "psubusb %%mm3, %%mm0  \n\t"\
+        "psubusb %%mm4, %%mm3  \n\t"\
+        "pminub  %%mm7, %%mm0  \n\t"\
+        "pminub  %%mm7, %%mm3  \n\t"\
+        "paddusb %%mm0, %%mm1  \n\t"\
+        "paddusb %%mm3, %%mm2  \n\t"\
+        "psubusb %%mm3, %%mm1  \n\t"\
+        "psubusb %%mm0, %%mm2  \n\t"
+
+// in: mm0=p1 mm1=p0 mm2=q0 mm3=q1 mm7=(tc&mask) %8=mm_bone
+// out: (q1addr) = clip( (q2+((p0+q0+1)>>1))>>1, q1-tc0, q1+tc0 )
+// clobbers: q2, tmp, tc0
+#define H264_DEBLOCK_Q1(p1, q2, q2addr, q1addr, tc0, tmp)\
+        "movq     %%mm1,  "#tmp"   \n\t"\
+        "pavgb    %%mm2,  "#tmp"   \n\t"\
+        "pavgb    "#tmp", "#q2"    \n\t" /* avg(p2,avg(p0,q0)) */\
+        "pxor   "q2addr", "#tmp"   \n\t"\
+        "pand     %8,     "#tmp"   \n\t" /* (p2^avg(p0,q0))&1 */\
+        "psubusb  "#tmp", "#q2"    \n\t" /* (p2+((p0+q0+1)>>1))>>1 */\
+        "movq     "#p1",  "#tmp"   \n\t"\
+        "psubusb  "#tc0", "#tmp"   \n\t"\
+        "paddusb  "#p1",  "#tc0"   \n\t"\
+        "pmaxub   "#tmp", "#q2"    \n\t"\
+        "pminub   "#tc0", "#q2"    \n\t"\
+        "movq     "#q2",  "q1addr" \n\t"
+
+static inline void h264_loop_filter_luma_mmx2(uint8_t *pix, int stride, int alpha1, int beta1, int8_t *tc0)
 {
-    uint64_t tmp0, tmp1;
+    uint64_t tmp0;
+    uint64_t tc = (uint8_t)tc0[1]*0x01010000 | (uint8_t)tc0[0]*0x0101;
+    // with luma, tc0=0 doesn't mean no filtering, so we need a separate input mask
+    uint32_t mask[2] = { (tc0[0]>=0)*0xffffffff, (tc0[1]>=0)*0xffffffff };
+
     asm volatile(
-        "movd  (%2,%4),   %%mm0        \n\t" //p1
-        "movd  (%2,%4,2), %%mm1        \n\t" //p0
-        "movd  (%3),      %%mm2        \n\t" //q0
-        "movd  (%3,%4),   %%mm3        \n\t" //q1
-        H264_DEBLOCK_THRESH(%6,%7)
-        "movq      %%mm7, %0           \n\t"
-
-// filter p1 if ABS(p2-p0) < beta
-        "movd      (%2),  %%mm3        \n\t"
-        "pxor      %%mm6, %%mm6        \n\t"
-        "punpcklbw %%mm6, %%mm3        \n\t" //p2
-        MMABS_DIFF_MMX2(%%mm1, %%mm3, %%mm5, %%mm6)
-        "pcmpgtw   %%mm5, %%mm4        \n\t"
-        "pand      %%mm7, %%mm4        \n\t" // mm4 = ( ABS( p2 - p0 ) < beta && filterp )
-        "movd      %5,    %%mm6        \n\t"
-        "pshufw    $0, %%mm6, %%mm6    \n\t" //tc
-
-        H264_DEBLOCK_DELTA_PQ1(%%mm0, %%mm3, %%mm7) // delta = clip( ( p2 + ((p0+q0+1)>>1) ) >> 1 ) - p1 )
-        "pand      %%mm4, %%mm5        \n\t"
-        "paddw     %%mm0, %%mm5        \n\t"
-        "packuswb  %%mm5, %%mm5        \n\t"
-        "movd      %%mm5, (%2,%4)      \n\t" // *p1 += delta
-        "psrlw     $15, %%mm4          \n\t"
-        "paddw     %%mm6, %%mm4        \n\t" // tc++
-        "movq      %%mm4, %1           \n\t"
-
-// filter q1 if ABS(q2-q0) < beta
-        "pxor      %%mm7, %%mm7        \n\t"
-        "movd  (%3,%4),   %%mm3        \n\t" //q1
-        "movd  (%3,%4,2), %%mm4        \n\t" //q2
-        "punpcklbw %%mm7, %%mm3        \n\t"
-        "punpcklbw %%mm7, %%mm4        \n\t"
-        MMABS_DIFF_MMX2(%%mm2, %%mm4, %%mm5, %%mm7)
-        "movd      %7,    %%mm7        \n\t"
-        "pshufw    $0, %%mm7, %%mm7    \n\t"
-        "pcmpgtw   %%mm5, %%mm7        \n\t"
-
-        H264_DEBLOCK_DELTA_PQ1(%%mm3, %%mm4, %%mm4) // delta = clip( ( q2 + ((p0+q0+1)>>1) ) >> 1 ) - q1 )
-        "movq      %0,    %%mm4        \n\t"
-        "pand      %%mm4, %%mm7        \n\t" // mm7 = ( ABS( q2 - q0 ) < beta && filterp )
-        "pand      %%mm7, %%mm5        \n\t"
-        "paddw     %%mm3, %%mm5        \n\t"
-        "packuswb  %%mm5, %%mm5        \n\t"
-        "movd      %%mm5, (%3,%4)      \n\t" // *q1 += delta
-        "movq      %1, %%mm6           \n\t"
-        "psrlw     $15, %%mm7          \n\t"
-        "paddw     %%mm7, %%mm6        \n\t" // tc++
-        "movq      %0,    %%mm4        \n\t"
-        "pand      %%mm4, %%mm6        \n\t"
-
-        H264_DEBLOCK_P0_Q0(%8)
-        "packuswb  %%mm1, %%mm1        \n\t"
-        "packuswb  %%mm2, %%mm2        \n\t"
-        "movd      %%mm1, (%2,%4,2)    \n\t"
-        "movd      %%mm2, (%3)         \n\t"
-
-        : "=m"(tmp0), "=m"(tmp1)
+        "movq    (%1,%3), %%mm0    \n\t" //p1
+        "movq    (%1,%3,2), %%mm1  \n\t" //p0
+        "movq    (%2),    %%mm2    \n\t" //q0
+        "movq    (%2,%3), %%mm3    \n\t" //q1
+        H264_DEBLOCK_MASK(%6, %7)
+        "pand     %5,     %%mm7    \n\t"
+        "movq     %%mm7,  %0       \n\t"
+
+        /* filter p1 */
+        "movq     (%1),   %%mm3    \n\t" //p2
+        DIFF_GT_MMX(%%mm1, %%mm3, %%mm5, %%mm6, %%mm4) // |p2-p0|>beta-1
+        "pandn    %%mm7,  %%mm6    \n\t"
+        "pcmpeqb  %%mm7,  %%mm6    \n\t"
+        "pand     %%mm7,  %%mm6    \n\t" // mask & |p2-p0|<beta
+        "pshufw  $80, %4, %%mm4    \n\t"
+        "pand     %%mm7,  %%mm4    \n\t" // mask & tc0
+        "movq     %8,     %%mm7    \n\t"
+        "pand     %%mm6,  %%mm7    \n\t" // mask & |p2-p0|<beta & 1
+        "pand     %%mm4,  %%mm6    \n\t" // mask & |p2-p0|<beta & tc0
+        "paddb    %%mm4,  %%mm7    \n\t" // tc++
+        H264_DEBLOCK_Q1(%%mm0, %%mm3, "(%1)", "(%1,%3)", %%mm6, %%mm4)
+
+        /* filter q1 */
+        "movq    (%2,%3,2), %%mm4  \n\t" //q2
+        DIFF_GT_MMX(%%mm2, %%mm4, %%mm5, %%mm6, %%mm3) // |q2-q0|>beta-1
+        "pandn    %0,     %%mm6    \n\t"
+        "pcmpeqb  %0,     %%mm6    \n\t"
+        "pand     %0,     %%mm6    \n\t"
+        "pshufw  $80, %4, %%mm5    \n\t"
+        "pand     %%mm6,  %%mm5    \n\t"
+        "pand     %8,     %%mm6    \n\t"
+        "paddb    %%mm6,  %%mm7    \n\t"
+        "movq    (%2,%3), %%mm3    \n\t"
+        H264_DEBLOCK_Q1(%%mm3, %%mm4, "(%2,%3,2)", "(%2,%3)", %%mm5, %%mm6)
+
+        /* filter p0, q0 */
+        H264_DEBLOCK_P0_Q0(%8, %9)
+        "movq      %%mm1, (%1,%3,2) \n\t"
+        "movq      %%mm2, (%2)      \n\t"
+
+        : "=m"(tmp0)
         : "r"(pix-3*stride), "r"(pix), "r"((long)stride),
-          "r"(tc0), "r"(alpha), "r"(beta), "m"(ff_pw_4)
+          "m"(tc), "m"(*(uint64_t*)mask), "m"(alpha1), "m"(beta1),
+          "m"(mm_bone), "m"(ff_pb_3F)
     );
 }
 
-static void h264_v_loop_filter_luma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int *tc0)
+static void h264_v_loop_filter_luma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0)
 {
-    int i;
-    for(i=0; i<4; i++, pix+=4) {
-        if(tc0[i] < 0)
-            continue;
-        h264_loop_filter_luma_mmx2(pix, stride, alpha, beta, tc0[i]);
-    }
+    if((tc0[0] & tc0[1]) >= 0)
+        h264_loop_filter_luma_mmx2(pix, stride, alpha-1, beta-1, tc0);
+    if((tc0[2] & tc0[3]) >= 0)
+        h264_loop_filter_luma_mmx2(pix+8, stride, alpha-1, beta-1, tc0+2);
 }
-
-static void h264_h_loop_filter_luma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int *tc0)
+static void h264_h_loop_filter_luma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0)
 {
-    uint8_t trans[4*8];
+    //FIXME: could cut some load/stores by merging transpose with filter
+    // also, it only needs to transpose 6x8
+    uint8_t trans[8*8];
     int i;
-    for(i=0; i<4; i++, pix+=4*stride) {
-        if(tc0[i] < 0)
+    for(i=0; i<2; i++, pix+=8*stride, tc0+=2) {
+        if((tc0[0] & tc0[1]) < 0)
             continue;
-        //FIXME: could cut some load/stores by merging transpose with filter
-        transpose4x4(trans, pix-4, 4, stride);
-        transpose4x4(trans+4*4, pix, 4, stride);
-        h264_loop_filter_luma_mmx2(trans+4*4, 4, alpha, beta, tc0[i]);
-        transpose4x4(pix-2, trans+2*4, stride, 4);
+        transpose4x4(trans,       pix-4,          8, stride);
+        transpose4x4(trans  +4*8, pix,            8, stride);
+        transpose4x4(trans+4,     pix-4+4*stride, 8, stride);
+        transpose4x4(trans+4+4*8, pix  +4*stride, 8, stride);
+        h264_loop_filter_luma_mmx2(trans+4*8, 8, alpha-1, beta-1, tc0);
+        transpose4x4(pix-2,          trans  +2*8, stride, 8);
+        transpose4x4(pix-2+4*stride, trans+4+2*8, stride, 8);
     }
 }
 
-static inline void h264_loop_filter_chroma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int *tc0)
+static inline void h264_loop_filter_chroma_mmx2(uint8_t *pix, int stride, int alpha1, int beta1, int8_t *tc0)
 {
     asm volatile(
-        "movd    (%0),    %%mm0     \n\t"
-        "movd    (%0,%2), %%mm1     \n\t"
-        "movd    (%1),    %%mm2     \n\t"
-        "movd    (%1,%2), %%mm3     \n\t"
-        H264_DEBLOCK_THRESH(%4,%5)
+        "movq    (%0),    %%mm0     \n\t" //p1
+        "movq    (%0,%2), %%mm1     \n\t" //p0
+        "movq    (%1),    %%mm2     \n\t" //q0
+        "movq    (%1,%2), %%mm3     \n\t" //q1
+        H264_DEBLOCK_MASK(%4, %5)
         "movd      %3,    %%mm6     \n\t"
-        "pshufw $0x50, %%mm6, %%mm6 \n\t" // mm6 = tc[1], tc[1], tc[0], tc[0]
-        "pand      %%mm7, %%mm6     \n\t"
-        H264_DEBLOCK_P0_Q0(%6)
-        "packuswb  %%mm1, %%mm1     \n\t"
-        "packuswb  %%mm2, %%mm2     \n\t"
-        "movd      %%mm1, (%0,%2)   \n\t"
-        "movd      %%mm2, (%1)      \n\t"
+        "punpcklbw %%mm6, %%mm6     \n\t"
+        "pand      %%mm6, %%mm7     \n\t" // mm7 = tc&mask
+        H264_DEBLOCK_P0_Q0(%6, %7)
+        "movq      %%mm1, (%0,%2)   \n\t"
+        "movq      %%mm2, (%1)      \n\t"
+
         :: "r"(pix-2*stride), "r"(pix), "r"((long)stride),
-           "r"(tc0[1]<<16 | tc0[0]),
-           "r"(alpha), "g"(beta), "m"(ff_pw_4)
+           "r"(*(uint32_t*)tc0),
+           "m"(alpha1), "m"(beta1), "m"(mm_bone), "m"(ff_pb_3F)
     );
 }
 
-static void h264_v_loop_filter_chroma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int *tc0)
+static void h264_v_loop_filter_chroma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0)
 {
-    int i;
-    for(i=0; i<2; i++) {
-        h264_loop_filter_chroma_mmx2(pix, stride, alpha, beta, tc0);
-        pix += 4;
-        tc0 += 2;
-    }
+    h264_loop_filter_chroma_mmx2(pix, stride, alpha-1, beta-1, tc0);
 }
 
-static void h264_h_loop_filter_chroma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int *tc0)
+static void h264_h_loop_filter_chroma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0)
 {
-    uint8_t trans[4*4];
-    int i;
-    for(i=0; i<2; i++) {
-        //FIXME: could cut some load/stores by merging transpose with filter
-        transpose4x4(trans, pix-2, 4, stride);
-        h264_loop_filter_chroma_mmx2(trans+2*4, 4, alpha, beta, tc0);
-        transpose4x4(pix-2, trans, stride, 4);
-        pix += 4*stride;
-        tc0 += 2;
-    }
+    //FIXME: could cut some load/stores by merging transpose with filter
+    uint8_t trans[8*4];
+    transpose4x4(trans, pix-2, 8, stride);
+    transpose4x4(trans+4, pix-2+4*stride, 8, stride);
+    h264_loop_filter_chroma_mmx2(trans+2*8, 8, alpha-1, beta-1, tc0);
+    transpose4x4(pix-2, trans, stride, 8);
+    transpose4x4(pix-2+4*stride, trans+4, stride, 8);
 }
 
+// p0 = (p0 + q1 + 2*p1 + 2) >> 2
+#define H264_FILTER_CHROMA4(p0, p1, q1, one) \
+    "movq    "#p0", %%mm4  \n\t"\
+    "pxor    "#q1", %%mm4  \n\t"\
+    "pand   "#one", %%mm4  \n\t" /* mm4 = (p0^q1)&1 */\
+    "pavgb   "#q1", "#p0"  \n\t"\
+    "psubusb %%mm4, "#p0"  \n\t"\
+    "pavgb   "#p1", "#p0"  \n\t" /* dst = avg(p1, avg(p0,q1) - ((p0^q1)&1)) */\
+
+static inline void h264_loop_filter_chroma_intra_mmx2(uint8_t *pix, int stride, int alpha1, int beta1)
+{
+    asm volatile(
+        "movq    (%0),    %%mm0     \n\t"
+        "movq    (%0,%2), %%mm1     \n\t"
+        "movq    (%1),    %%mm2     \n\t"
+        "movq    (%1,%2), %%mm3     \n\t"
+        H264_DEBLOCK_MASK(%3, %4)
+        "movq    %%mm1,   %%mm5     \n\t"
+        "movq    %%mm2,   %%mm6     \n\t"
+        H264_FILTER_CHROMA4(%%mm1, %%mm0, %%mm3, %5) //p0'
+        H264_FILTER_CHROMA4(%%mm2, %%mm3, %%mm0, %5) //q0'
+        "psubb   %%mm5,   %%mm1     \n\t"
+        "psubb   %%mm6,   %%mm2     \n\t"
+        "pand    %%mm7,   %%mm1     \n\t"
+        "pand    %%mm7,   %%mm2     \n\t"
+        "paddb   %%mm5,   %%mm1     \n\t"
+        "paddb   %%mm6,   %%mm2     \n\t"
+        "movq    %%mm1,   (%0,%2)   \n\t"
+        "movq    %%mm2,   (%1)      \n\t"
+        :: "r"(pix-2*stride), "r"(pix), "r"((long)stride),
+           "m"(alpha1), "m"(beta1), "m"(mm_bone)
+    );
+}
+
+static void h264_v_loop_filter_chroma_intra_mmx2(uint8_t *pix, int stride, int alpha, int beta)
+{
+    h264_loop_filter_chroma_intra_mmx2(pix, stride, alpha-1, beta-1);
+}
+
+static void h264_h_loop_filter_chroma_intra_mmx2(uint8_t *pix, int stride, int alpha, int beta)
+{
+    //FIXME: could cut some load/stores by merging transpose with filter
+    uint8_t trans[8*4];
+    transpose4x4(trans, pix-2, 8, stride);
+    transpose4x4(trans+4, pix-2+4*stride, 8, stride);
+    h264_loop_filter_chroma_intra_mmx2(trans+2*8, 8, alpha-1, beta-1);
+    transpose4x4(pix-2, trans, stride, 8);
+    transpose4x4(pix-2+4*stride, trans+4, stride, 8);
+}
+
+
 #ifdef CONFIG_ENCODERS
 static int pix_norm1_mmx(uint8_t *pix, int line_size) {
     int tmp;
@@ -3415,6 +3477,8 @@ void dsputil_init_mmx(DSPContext* c, AVCodecContext *avctx)
             c->h264_h_loop_filter_luma= h264_h_loop_filter_luma_mmx2;
             c->h264_v_loop_filter_chroma= h264_v_loop_filter_chroma_mmx2;
             c->h264_h_loop_filter_chroma= h264_h_loop_filter_chroma_mmx2;
+            c->h264_v_loop_filter_chroma_intra= h264_v_loop_filter_chroma_intra_mmx2;
+            c->h264_h_loop_filter_chroma_intra= h264_h_loop_filter_chroma_intra_mmx2;
 
 #ifdef CONFIG_ENCODERS
             c->sub_hfyu_median_prediction= sub_hfyu_median_prediction_mmx2;