/* * software RGB to RGB converter * pluralize by software PAL8 to RGB converter * software YUV to YUV converter * software YUV to RGB converter * Written by Nick Kurshev. * palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at) * lot of big-endian byte order fixes by Alex Beregszaszi * * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * * The C code (not assembly, MMX, ...) of this file can be used * under the LGPL license. */ #include #undef PREFETCH #undef MOVNTQ #undef EMMS #undef SFENCE #undef MMREG_SIZE #undef PREFETCHW #undef PAVGB #if HAVE_SSE2 #define MMREG_SIZE 16 #else #define MMREG_SIZE 8 #endif #if HAVE_AMD3DNOW #define PREFETCH "prefetch" #define PREFETCHW "prefetchw" #define PAVGB "pavgusb" #elif HAVE_MMX2 #define PREFETCH "prefetchnta" #define PREFETCHW "prefetcht0" #define PAVGB "pavgb" #else #define PREFETCH " # nop" #define PREFETCHW " # nop" #endif #if HAVE_AMD3DNOW /* On K6 femms is faster than emms. On K7 femms is directly mapped to emms. */ #define EMMS "femms" #else #define EMMS "emms" #endif #if HAVE_MMX2 #define MOVNTQ "movntq" #define SFENCE "sfence" #else #define MOVNTQ "movq" #define SFENCE " # nop" #endif static inline void RENAME(rgb24tobgr32)(const uint8_t *src, uint8_t *dst, long src_size) { uint8_t *dest = dst; const uint8_t *s = src; const uint8_t *end; #if HAVE_MMX const uint8_t *mm_end; #endif end = s + src_size; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory"); mm_end = end - 23; __asm__ volatile("movq %0, %%mm7"::"m"(mask32a):"memory"); while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movd %1, %%mm0 \n\t" "punpckldq 3%1, %%mm0 \n\t" "movd 6%1, %%mm1 \n\t" "punpckldq 9%1, %%mm1 \n\t" "movd 12%1, %%mm2 \n\t" "punpckldq 15%1, %%mm2 \n\t" "movd 18%1, %%mm3 \n\t" "punpckldq 21%1, %%mm3 \n\t" "por %%mm7, %%mm0 \n\t" "por %%mm7, %%mm1 \n\t" "por %%mm7, %%mm2 \n\t" "por %%mm7, %%mm3 \n\t" MOVNTQ" %%mm0, %0 \n\t" MOVNTQ" %%mm1, 8%0 \n\t" MOVNTQ" %%mm2, 16%0 \n\t" MOVNTQ" %%mm3, 24%0" :"=m"(*dest) :"m"(*s) :"memory"); dest += 32; s += 24; } __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { #ifdef WORDS_BIGENDIAN /* RGB24 (= R,G,B) -> RGB32 (= A,B,G,R) */ *dest++ = 255; *dest++ = s[2]; *dest++ = s[1]; *dest++ = s[0]; s+=3; #else *dest++ = *s++; *dest++ = *s++; *dest++ = *s++; *dest++ = 255; #endif } } static inline void RENAME(rgb32tobgr24)(const uint8_t *src, uint8_t *dst, long src_size) { uint8_t *dest = dst; const uint8_t *s = src; const uint8_t *end; #if HAVE_MMX const uint8_t *mm_end; #endif end = s + src_size; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory"); mm_end = end - 31; while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movq %1, %%mm0 \n\t" "movq 8%1, %%mm1 \n\t" "movq 16%1, %%mm4 \n\t" "movq 24%1, %%mm5 \n\t" "movq %%mm0, %%mm2 \n\t" "movq %%mm1, %%mm3 \n\t" "movq %%mm4, %%mm6 \n\t" "movq %%mm5, %%mm7 \n\t" "psrlq $8, %%mm2 \n\t" "psrlq $8, %%mm3 \n\t" "psrlq $8, %%mm6 \n\t" "psrlq $8, %%mm7 \n\t" "pand %2, %%mm0 \n\t" "pand %2, %%mm1 \n\t" "pand %2, %%mm4 \n\t" "pand %2, %%mm5 \n\t" "pand %3, %%mm2 \n\t" "pand %3, %%mm3 \n\t" "pand %3, %%mm6 \n\t" "pand %3, %%mm7 \n\t" "por %%mm2, %%mm0 \n\t" "por %%mm3, %%mm1 \n\t" "por %%mm6, %%mm4 \n\t" "por %%mm7, %%mm5 \n\t" "movq %%mm1, %%mm2 \n\t" "movq %%mm4, %%mm3 \n\t" "psllq $48, %%mm2 \n\t" "psllq $32, %%mm3 \n\t" "pand %4, %%mm2 \n\t" "pand %5, %%mm3 \n\t" "por %%mm2, %%mm0 \n\t" "psrlq $16, %%mm1 \n\t" "psrlq $32, %%mm4 \n\t" "psllq $16, %%mm5 \n\t" "por %%mm3, %%mm1 \n\t" "pand %6, %%mm5 \n\t" "por %%mm5, %%mm4 \n\t" MOVNTQ" %%mm0, %0 \n\t" MOVNTQ" %%mm1, 8%0 \n\t" MOVNTQ" %%mm4, 16%0" :"=m"(*dest) :"m"(*s),"m"(mask24l), "m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh) :"memory"); dest += 24; s += 32; } __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { #ifdef WORDS_BIGENDIAN /* RGB32 (= A,B,G,R) -> RGB24 (= R,G,B) */ s++; dest[2] = *s++; dest[1] = *s++; dest[0] = *s++; dest += 3; #else *dest++ = *s++; *dest++ = *s++; *dest++ = *s++; s++; #endif } } /* original by Strepto/Astral ported to gcc & bugfixed: A'rpi MMX2, 3DNOW optimization by Nick Kurshev 32-bit C version, and and&add trick by Michael Niedermayer */ static inline void RENAME(rgb15to16)(const uint8_t *src, uint8_t *dst, long src_size) { register const uint8_t* s=src; register uint8_t* d=dst; register const uint8_t *end; const uint8_t *mm_end; end = s + src_size; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*s)); __asm__ volatile("movq %0, %%mm4"::"m"(mask15s)); mm_end = end - 15; while (s>1)&0x7FE07FE0) | (x&0x001F001F); s+=4; d+=4; } if (s < end) { register uint16_t x= *((const uint16_t*)s); *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F); s+=2; d+=2; } } static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, long src_size) { const uint8_t *s = src; const uint8_t *end; #if HAVE_MMX const uint8_t *mm_end; #endif uint16_t *d = (uint16_t *)dst; end = s + src_size; #if HAVE_MMX mm_end = end - 15; #if 1 //is faster only if multiplies are reasonably fast (FIXME figure out on which CPUs this is faster, on Athlon it is slightly faster) __asm__ volatile( "movq %3, %%mm5 \n\t" "movq %4, %%mm6 \n\t" "movq %5, %%mm7 \n\t" "jmp 2f \n\t" ASMALIGN(4) "1: \n\t" PREFETCH" 32(%1) \n\t" "movd (%1), %%mm0 \n\t" "movd 4(%1), %%mm3 \n\t" "punpckldq 8(%1), %%mm0 \n\t" "punpckldq 12(%1), %%mm3 \n\t" "movq %%mm0, %%mm1 \n\t" "movq %%mm3, %%mm4 \n\t" "pand %%mm6, %%mm0 \n\t" "pand %%mm6, %%mm3 \n\t" "pmaddwd %%mm7, %%mm0 \n\t" "pmaddwd %%mm7, %%mm3 \n\t" "pand %%mm5, %%mm1 \n\t" "pand %%mm5, %%mm4 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm4, %%mm3 \n\t" "psrld $5, %%mm0 \n\t" "pslld $11, %%mm3 \n\t" "por %%mm3, %%mm0 \n\t" MOVNTQ" %%mm0, (%0) \n\t" "add $16, %1 \n\t" "add $8, %0 \n\t" "2: \n\t" "cmp %2, %1 \n\t" " jb 1b \n\t" : "+r" (d), "+r"(s) : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216) ); #else __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory"); __asm__ volatile( "movq %0, %%mm7 \n\t" "movq %1, %%mm6 \n\t" ::"m"(red_16mask),"m"(green_16mask)); while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movd %1, %%mm0 \n\t" "movd 4%1, %%mm3 \n\t" "punpckldq 8%1, %%mm0 \n\t" "punpckldq 12%1, %%mm3 \n\t" "movq %%mm0, %%mm1 \n\t" "movq %%mm0, %%mm2 \n\t" "movq %%mm3, %%mm4 \n\t" "movq %%mm3, %%mm5 \n\t" "psrlq $3, %%mm0 \n\t" "psrlq $3, %%mm3 \n\t" "pand %2, %%mm0 \n\t" "pand %2, %%mm3 \n\t" "psrlq $5, %%mm1 \n\t" "psrlq $5, %%mm4 \n\t" "pand %%mm6, %%mm1 \n\t" "pand %%mm6, %%mm4 \n\t" "psrlq $8, %%mm2 \n\t" "psrlq $8, %%mm5 \n\t" "pand %%mm7, %%mm2 \n\t" "pand %%mm7, %%mm5 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm4, %%mm3 \n\t" "por %%mm2, %%mm0 \n\t" "por %%mm5, %%mm3 \n\t" "psllq $16, %%mm3 \n\t" "por %%mm3, %%mm0 \n\t" MOVNTQ" %%mm0, %0 \n\t" :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory"); d += 4; s += 16; } #endif __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { register int rgb = *(const uint32_t*)s; s += 4; *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8); } } static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, long src_size) { const uint8_t *s = src; const uint8_t *end; #if HAVE_MMX const uint8_t *mm_end; #endif uint16_t *d = (uint16_t *)dst; end = s + src_size; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory"); __asm__ volatile( "movq %0, %%mm7 \n\t" "movq %1, %%mm6 \n\t" ::"m"(red_16mask),"m"(green_16mask)); mm_end = end - 15; while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movd %1, %%mm0 \n\t" "movd 4%1, %%mm3 \n\t" "punpckldq 8%1, %%mm0 \n\t" "punpckldq 12%1, %%mm3 \n\t" "movq %%mm0, %%mm1 \n\t" "movq %%mm0, %%mm2 \n\t" "movq %%mm3, %%mm4 \n\t" "movq %%mm3, %%mm5 \n\t" "psllq $8, %%mm0 \n\t" "psllq $8, %%mm3 \n\t" "pand %%mm7, %%mm0 \n\t" "pand %%mm7, %%mm3 \n\t" "psrlq $5, %%mm1 \n\t" "psrlq $5, %%mm4 \n\t" "pand %%mm6, %%mm1 \n\t" "pand %%mm6, %%mm4 \n\t" "psrlq $19, %%mm2 \n\t" "psrlq $19, %%mm5 \n\t" "pand %2, %%mm2 \n\t" "pand %2, %%mm5 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm4, %%mm3 \n\t" "por %%mm2, %%mm0 \n\t" "por %%mm5, %%mm3 \n\t" "psllq $16, %%mm3 \n\t" "por %%mm3, %%mm0 \n\t" MOVNTQ" %%mm0, %0 \n\t" :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory"); d += 4; s += 16; } __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { register int rgb = *(const uint32_t*)s; s += 4; *d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19); } } static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, long src_size) { const uint8_t *s = src; const uint8_t *end; #if HAVE_MMX const uint8_t *mm_end; #endif uint16_t *d = (uint16_t *)dst; end = s + src_size; #if HAVE_MMX mm_end = end - 15; #if 1 //is faster only if multiplies are reasonably fast (FIXME figure out on which CPUs this is faster, on Athlon it is slightly faster) __asm__ volatile( "movq %3, %%mm5 \n\t" "movq %4, %%mm6 \n\t" "movq %5, %%mm7 \n\t" "jmp 2f \n\t" ASMALIGN(4) "1: \n\t" PREFETCH" 32(%1) \n\t" "movd (%1), %%mm0 \n\t" "movd 4(%1), %%mm3 \n\t" "punpckldq 8(%1), %%mm0 \n\t" "punpckldq 12(%1), %%mm3 \n\t" "movq %%mm0, %%mm1 \n\t" "movq %%mm3, %%mm4 \n\t" "pand %%mm6, %%mm0 \n\t" "pand %%mm6, %%mm3 \n\t" "pmaddwd %%mm7, %%mm0 \n\t" "pmaddwd %%mm7, %%mm3 \n\t" "pand %%mm5, %%mm1 \n\t" "pand %%mm5, %%mm4 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm4, %%mm3 \n\t" "psrld $6, %%mm0 \n\t" "pslld $10, %%mm3 \n\t" "por %%mm3, %%mm0 \n\t" MOVNTQ" %%mm0, (%0) \n\t" "add $16, %1 \n\t" "add $8, %0 \n\t" "2: \n\t" "cmp %2, %1 \n\t" " jb 1b \n\t" : "+r" (d), "+r"(s) : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215) ); #else __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory"); __asm__ volatile( "movq %0, %%mm7 \n\t" "movq %1, %%mm6 \n\t" ::"m"(red_15mask),"m"(green_15mask)); while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movd %1, %%mm0 \n\t" "movd 4%1, %%mm3 \n\t" "punpckldq 8%1, %%mm0 \n\t" "punpckldq 12%1, %%mm3 \n\t" "movq %%mm0, %%mm1 \n\t" "movq %%mm0, %%mm2 \n\t" "movq %%mm3, %%mm4 \n\t" "movq %%mm3, %%mm5 \n\t" "psrlq $3, %%mm0 \n\t" "psrlq $3, %%mm3 \n\t" "pand %2, %%mm0 \n\t" "pand %2, %%mm3 \n\t" "psrlq $6, %%mm1 \n\t" "psrlq $6, %%mm4 \n\t" "pand %%mm6, %%mm1 \n\t" "pand %%mm6, %%mm4 \n\t" "psrlq $9, %%mm2 \n\t" "psrlq $9, %%mm5 \n\t" "pand %%mm7, %%mm2 \n\t" "pand %%mm7, %%mm5 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm4, %%mm3 \n\t" "por %%mm2, %%mm0 \n\t" "por %%mm5, %%mm3 \n\t" "psllq $16, %%mm3 \n\t" "por %%mm3, %%mm0 \n\t" MOVNTQ" %%mm0, %0 \n\t" :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory"); d += 4; s += 16; } #endif __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { register int rgb = *(const uint32_t*)s; s += 4; *d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9); } } static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, long src_size) { const uint8_t *s = src; const uint8_t *end; #if HAVE_MMX const uint8_t *mm_end; #endif uint16_t *d = (uint16_t *)dst; end = s + src_size; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory"); __asm__ volatile( "movq %0, %%mm7 \n\t" "movq %1, %%mm6 \n\t" ::"m"(red_15mask),"m"(green_15mask)); mm_end = end - 15; while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movd %1, %%mm0 \n\t" "movd 4%1, %%mm3 \n\t" "punpckldq 8%1, %%mm0 \n\t" "punpckldq 12%1, %%mm3 \n\t" "movq %%mm0, %%mm1 \n\t" "movq %%mm0, %%mm2 \n\t" "movq %%mm3, %%mm4 \n\t" "movq %%mm3, %%mm5 \n\t" "psllq $7, %%mm0 \n\t" "psllq $7, %%mm3 \n\t" "pand %%mm7, %%mm0 \n\t" "pand %%mm7, %%mm3 \n\t" "psrlq $6, %%mm1 \n\t" "psrlq $6, %%mm4 \n\t" "pand %%mm6, %%mm1 \n\t" "pand %%mm6, %%mm4 \n\t" "psrlq $19, %%mm2 \n\t" "psrlq $19, %%mm5 \n\t" "pand %2, %%mm2 \n\t" "pand %2, %%mm5 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm4, %%mm3 \n\t" "por %%mm2, %%mm0 \n\t" "por %%mm5, %%mm3 \n\t" "psllq $16, %%mm3 \n\t" "por %%mm3, %%mm0 \n\t" MOVNTQ" %%mm0, %0 \n\t" :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory"); d += 4; s += 16; } __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { register int rgb = *(const uint32_t*)s; s += 4; *d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19); } } static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, long src_size) { const uint8_t *s = src; const uint8_t *end; #if HAVE_MMX const uint8_t *mm_end; #endif uint16_t *d = (uint16_t *)dst; end = s + src_size; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory"); __asm__ volatile( "movq %0, %%mm7 \n\t" "movq %1, %%mm6 \n\t" ::"m"(red_16mask),"m"(green_16mask)); mm_end = end - 11; while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movd %1, %%mm0 \n\t" "movd 3%1, %%mm3 \n\t" "punpckldq 6%1, %%mm0 \n\t" "punpckldq 9%1, %%mm3 \n\t" "movq %%mm0, %%mm1 \n\t" "movq %%mm0, %%mm2 \n\t" "movq %%mm3, %%mm4 \n\t" "movq %%mm3, %%mm5 \n\t" "psrlq $3, %%mm0 \n\t" "psrlq $3, %%mm3 \n\t" "pand %2, %%mm0 \n\t" "pand %2, %%mm3 \n\t" "psrlq $5, %%mm1 \n\t" "psrlq $5, %%mm4 \n\t" "pand %%mm6, %%mm1 \n\t" "pand %%mm6, %%mm4 \n\t" "psrlq $8, %%mm2 \n\t" "psrlq $8, %%mm5 \n\t" "pand %%mm7, %%mm2 \n\t" "pand %%mm7, %%mm5 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm4, %%mm3 \n\t" "por %%mm2, %%mm0 \n\t" "por %%mm5, %%mm3 \n\t" "psllq $16, %%mm3 \n\t" "por %%mm3, %%mm0 \n\t" MOVNTQ" %%mm0, %0 \n\t" :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory"); d += 4; s += 12; } __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { const int b = *s++; const int g = *s++; const int r = *s++; *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8); } } static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, long src_size) { const uint8_t *s = src; const uint8_t *end; #if HAVE_MMX const uint8_t *mm_end; #endif uint16_t *d = (uint16_t *)dst; end = s + src_size; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory"); __asm__ volatile( "movq %0, %%mm7 \n\t" "movq %1, %%mm6 \n\t" ::"m"(red_16mask),"m"(green_16mask)); mm_end = end - 15; while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movd %1, %%mm0 \n\t" "movd 3%1, %%mm3 \n\t" "punpckldq 6%1, %%mm0 \n\t" "punpckldq 9%1, %%mm3 \n\t" "movq %%mm0, %%mm1 \n\t" "movq %%mm0, %%mm2 \n\t" "movq %%mm3, %%mm4 \n\t" "movq %%mm3, %%mm5 \n\t" "psllq $8, %%mm0 \n\t" "psllq $8, %%mm3 \n\t" "pand %%mm7, %%mm0 \n\t" "pand %%mm7, %%mm3 \n\t" "psrlq $5, %%mm1 \n\t" "psrlq $5, %%mm4 \n\t" "pand %%mm6, %%mm1 \n\t" "pand %%mm6, %%mm4 \n\t" "psrlq $19, %%mm2 \n\t" "psrlq $19, %%mm5 \n\t" "pand %2, %%mm2 \n\t" "pand %2, %%mm5 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm4, %%mm3 \n\t" "por %%mm2, %%mm0 \n\t" "por %%mm5, %%mm3 \n\t" "psllq $16, %%mm3 \n\t" "por %%mm3, %%mm0 \n\t" MOVNTQ" %%mm0, %0 \n\t" :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory"); d += 4; s += 12; } __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { const int r = *s++; const int g = *s++; const int b = *s++; *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8); } } static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, long src_size) { const uint8_t *s = src; const uint8_t *end; #if HAVE_MMX const uint8_t *mm_end; #endif uint16_t *d = (uint16_t *)dst; end = s + src_size; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory"); __asm__ volatile( "movq %0, %%mm7 \n\t" "movq %1, %%mm6 \n\t" ::"m"(red_15mask),"m"(green_15mask)); mm_end = end - 11; while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movd %1, %%mm0 \n\t" "movd 3%1, %%mm3 \n\t" "punpckldq 6%1, %%mm0 \n\t" "punpckldq 9%1, %%mm3 \n\t" "movq %%mm0, %%mm1 \n\t" "movq %%mm0, %%mm2 \n\t" "movq %%mm3, %%mm4 \n\t" "movq %%mm3, %%mm5 \n\t" "psrlq $3, %%mm0 \n\t" "psrlq $3, %%mm3 \n\t" "pand %2, %%mm0 \n\t" "pand %2, %%mm3 \n\t" "psrlq $6, %%mm1 \n\t" "psrlq $6, %%mm4 \n\t" "pand %%mm6, %%mm1 \n\t" "pand %%mm6, %%mm4 \n\t" "psrlq $9, %%mm2 \n\t" "psrlq $9, %%mm5 \n\t" "pand %%mm7, %%mm2 \n\t" "pand %%mm7, %%mm5 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm4, %%mm3 \n\t" "por %%mm2, %%mm0 \n\t" "por %%mm5, %%mm3 \n\t" "psllq $16, %%mm3 \n\t" "por %%mm3, %%mm0 \n\t" MOVNTQ" %%mm0, %0 \n\t" :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory"); d += 4; s += 12; } __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { const int b = *s++; const int g = *s++; const int r = *s++; *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7); } } static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, long src_size) { const uint8_t *s = src; const uint8_t *end; #if HAVE_MMX const uint8_t *mm_end; #endif uint16_t *d = (uint16_t *)dst; end = s + src_size; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*src):"memory"); __asm__ volatile( "movq %0, %%mm7 \n\t" "movq %1, %%mm6 \n\t" ::"m"(red_15mask),"m"(green_15mask)); mm_end = end - 15; while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movd %1, %%mm0 \n\t" "movd 3%1, %%mm3 \n\t" "punpckldq 6%1, %%mm0 \n\t" "punpckldq 9%1, %%mm3 \n\t" "movq %%mm0, %%mm1 \n\t" "movq %%mm0, %%mm2 \n\t" "movq %%mm3, %%mm4 \n\t" "movq %%mm3, %%mm5 \n\t" "psllq $7, %%mm0 \n\t" "psllq $7, %%mm3 \n\t" "pand %%mm7, %%mm0 \n\t" "pand %%mm7, %%mm3 \n\t" "psrlq $6, %%mm1 \n\t" "psrlq $6, %%mm4 \n\t" "pand %%mm6, %%mm1 \n\t" "pand %%mm6, %%mm4 \n\t" "psrlq $19, %%mm2 \n\t" "psrlq $19, %%mm5 \n\t" "pand %2, %%mm2 \n\t" "pand %2, %%mm5 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm4, %%mm3 \n\t" "por %%mm2, %%mm0 \n\t" "por %%mm5, %%mm3 \n\t" "psllq $16, %%mm3 \n\t" "por %%mm3, %%mm0 \n\t" MOVNTQ" %%mm0, %0 \n\t" :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory"); d += 4; s += 12; } __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { const int r = *s++; const int g = *s++; const int b = *s++; *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7); } } /* I use less accurate approximation here by simply left-shifting the input value and filling the low order bits with zeroes. This method improves PNG compression but this scheme cannot reproduce white exactly, since it does not generate an all-ones maximum value; the net effect is to darken the image slightly. The better method should be "left bit replication": 4 3 2 1 0 --------- 1 1 0 1 1 7 6 5 4 3 2 1 0 ---------------- 1 1 0 1 1 1 1 0 |=======| |===| | leftmost bits repeated to fill open bits | original bits */ static inline void RENAME(rgb15tobgr24)(const uint8_t *src, uint8_t *dst, long src_size) { const uint16_t *end; #if HAVE_MMX const uint16_t *mm_end; #endif uint8_t *d = dst; const uint16_t *s = (const uint16_t*)src; end = s + src_size/2; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory"); mm_end = end - 7; while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movq %1, %%mm0 \n\t" "movq %1, %%mm1 \n\t" "movq %1, %%mm2 \n\t" "pand %2, %%mm0 \n\t" "pand %3, %%mm1 \n\t" "pand %4, %%mm2 \n\t" "psllq $3, %%mm0 \n\t" "psrlq $2, %%mm1 \n\t" "psrlq $7, %%mm2 \n\t" "movq %%mm0, %%mm3 \n\t" "movq %%mm1, %%mm4 \n\t" "movq %%mm2, %%mm5 \n\t" "punpcklwd %5, %%mm0 \n\t" "punpcklwd %5, %%mm1 \n\t" "punpcklwd %5, %%mm2 \n\t" "punpckhwd %5, %%mm3 \n\t" "punpckhwd %5, %%mm4 \n\t" "punpckhwd %5, %%mm5 \n\t" "psllq $8, %%mm1 \n\t" "psllq $16, %%mm2 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm2, %%mm0 \n\t" "psllq $8, %%mm4 \n\t" "psllq $16, %%mm5 \n\t" "por %%mm4, %%mm3 \n\t" "por %%mm5, %%mm3 \n\t" "movq %%mm0, %%mm6 \n\t" "movq %%mm3, %%mm7 \n\t" "movq 8%1, %%mm0 \n\t" "movq 8%1, %%mm1 \n\t" "movq 8%1, %%mm2 \n\t" "pand %2, %%mm0 \n\t" "pand %3, %%mm1 \n\t" "pand %4, %%mm2 \n\t" "psllq $3, %%mm0 \n\t" "psrlq $2, %%mm1 \n\t" "psrlq $7, %%mm2 \n\t" "movq %%mm0, %%mm3 \n\t" "movq %%mm1, %%mm4 \n\t" "movq %%mm2, %%mm5 \n\t" "punpcklwd %5, %%mm0 \n\t" "punpcklwd %5, %%mm1 \n\t" "punpcklwd %5, %%mm2 \n\t" "punpckhwd %5, %%mm3 \n\t" "punpckhwd %5, %%mm4 \n\t" "punpckhwd %5, %%mm5 \n\t" "psllq $8, %%mm1 \n\t" "psllq $16, %%mm2 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm2, %%mm0 \n\t" "psllq $8, %%mm4 \n\t" "psllq $16, %%mm5 \n\t" "por %%mm4, %%mm3 \n\t" "por %%mm5, %%mm3 \n\t" :"=m"(*d) :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r), "m"(mmx_null) :"memory"); /* borrowed 32 to 24 */ __asm__ volatile( "movq %%mm0, %%mm4 \n\t" "movq %%mm3, %%mm5 \n\t" "movq %%mm6, %%mm0 \n\t" "movq %%mm7, %%mm1 \n\t" "movq %%mm4, %%mm6 \n\t" "movq %%mm5, %%mm7 \n\t" "movq %%mm0, %%mm2 \n\t" "movq %%mm1, %%mm3 \n\t" "psrlq $8, %%mm2 \n\t" "psrlq $8, %%mm3 \n\t" "psrlq $8, %%mm6 \n\t" "psrlq $8, %%mm7 \n\t" "pand %2, %%mm0 \n\t" "pand %2, %%mm1 \n\t" "pand %2, %%mm4 \n\t" "pand %2, %%mm5 \n\t" "pand %3, %%mm2 \n\t" "pand %3, %%mm3 \n\t" "pand %3, %%mm6 \n\t" "pand %3, %%mm7 \n\t" "por %%mm2, %%mm0 \n\t" "por %%mm3, %%mm1 \n\t" "por %%mm6, %%mm4 \n\t" "por %%mm7, %%mm5 \n\t" "movq %%mm1, %%mm2 \n\t" "movq %%mm4, %%mm3 \n\t" "psllq $48, %%mm2 \n\t" "psllq $32, %%mm3 \n\t" "pand %4, %%mm2 \n\t" "pand %5, %%mm3 \n\t" "por %%mm2, %%mm0 \n\t" "psrlq $16, %%mm1 \n\t" "psrlq $32, %%mm4 \n\t" "psllq $16, %%mm5 \n\t" "por %%mm3, %%mm1 \n\t" "pand %6, %%mm5 \n\t" "por %%mm5, %%mm4 \n\t" MOVNTQ" %%mm0, %0 \n\t" MOVNTQ" %%mm1, 8%0 \n\t" MOVNTQ" %%mm4, 16%0" :"=m"(*d) :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh) :"memory"); d += 24; s += 8; } __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { register uint16_t bgr; bgr = *s++; *d++ = (bgr&0x1F)<<3; *d++ = (bgr&0x3E0)>>2; *d++ = (bgr&0x7C00)>>7; } } static inline void RENAME(rgb16tobgr24)(const uint8_t *src, uint8_t *dst, long src_size) { const uint16_t *end; #if HAVE_MMX const uint16_t *mm_end; #endif uint8_t *d = (uint8_t *)dst; const uint16_t *s = (const uint16_t *)src; end = s + src_size/2; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory"); mm_end = end - 7; while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movq %1, %%mm0 \n\t" "movq %1, %%mm1 \n\t" "movq %1, %%mm2 \n\t" "pand %2, %%mm0 \n\t" "pand %3, %%mm1 \n\t" "pand %4, %%mm2 \n\t" "psllq $3, %%mm0 \n\t" "psrlq $3, %%mm1 \n\t" "psrlq $8, %%mm2 \n\t" "movq %%mm0, %%mm3 \n\t" "movq %%mm1, %%mm4 \n\t" "movq %%mm2, %%mm5 \n\t" "punpcklwd %5, %%mm0 \n\t" "punpcklwd %5, %%mm1 \n\t" "punpcklwd %5, %%mm2 \n\t" "punpckhwd %5, %%mm3 \n\t" "punpckhwd %5, %%mm4 \n\t" "punpckhwd %5, %%mm5 \n\t" "psllq $8, %%mm1 \n\t" "psllq $16, %%mm2 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm2, %%mm0 \n\t" "psllq $8, %%mm4 \n\t" "psllq $16, %%mm5 \n\t" "por %%mm4, %%mm3 \n\t" "por %%mm5, %%mm3 \n\t" "movq %%mm0, %%mm6 \n\t" "movq %%mm3, %%mm7 \n\t" "movq 8%1, %%mm0 \n\t" "movq 8%1, %%mm1 \n\t" "movq 8%1, %%mm2 \n\t" "pand %2, %%mm0 \n\t" "pand %3, %%mm1 \n\t" "pand %4, %%mm2 \n\t" "psllq $3, %%mm0 \n\t" "psrlq $3, %%mm1 \n\t" "psrlq $8, %%mm2 \n\t" "movq %%mm0, %%mm3 \n\t" "movq %%mm1, %%mm4 \n\t" "movq %%mm2, %%mm5 \n\t" "punpcklwd %5, %%mm0 \n\t" "punpcklwd %5, %%mm1 \n\t" "punpcklwd %5, %%mm2 \n\t" "punpckhwd %5, %%mm3 \n\t" "punpckhwd %5, %%mm4 \n\t" "punpckhwd %5, %%mm5 \n\t" "psllq $8, %%mm1 \n\t" "psllq $16, %%mm2 \n\t" "por %%mm1, %%mm0 \n\t" "por %%mm2, %%mm0 \n\t" "psllq $8, %%mm4 \n\t" "psllq $16, %%mm5 \n\t" "por %%mm4, %%mm3 \n\t" "por %%mm5, %%mm3 \n\t" :"=m"(*d) :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null) :"memory"); /* borrowed 32 to 24 */ __asm__ volatile( "movq %%mm0, %%mm4 \n\t" "movq %%mm3, %%mm5 \n\t" "movq %%mm6, %%mm0 \n\t" "movq %%mm7, %%mm1 \n\t" "movq %%mm4, %%mm6 \n\t" "movq %%mm5, %%mm7 \n\t" "movq %%mm0, %%mm2 \n\t" "movq %%mm1, %%mm3 \n\t" "psrlq $8, %%mm2 \n\t" "psrlq $8, %%mm3 \n\t" "psrlq $8, %%mm6 \n\t" "psrlq $8, %%mm7 \n\t" "pand %2, %%mm0 \n\t" "pand %2, %%mm1 \n\t" "pand %2, %%mm4 \n\t" "pand %2, %%mm5 \n\t" "pand %3, %%mm2 \n\t" "pand %3, %%mm3 \n\t" "pand %3, %%mm6 \n\t" "pand %3, %%mm7 \n\t" "por %%mm2, %%mm0 \n\t" "por %%mm3, %%mm1 \n\t" "por %%mm6, %%mm4 \n\t" "por %%mm7, %%mm5 \n\t" "movq %%mm1, %%mm2 \n\t" "movq %%mm4, %%mm3 \n\t" "psllq $48, %%mm2 \n\t" "psllq $32, %%mm3 \n\t" "pand %4, %%mm2 \n\t" "pand %5, %%mm3 \n\t" "por %%mm2, %%mm0 \n\t" "psrlq $16, %%mm1 \n\t" "psrlq $32, %%mm4 \n\t" "psllq $16, %%mm5 \n\t" "por %%mm3, %%mm1 \n\t" "pand %6, %%mm5 \n\t" "por %%mm5, %%mm4 \n\t" MOVNTQ" %%mm0, %0 \n\t" MOVNTQ" %%mm1, 8%0 \n\t" MOVNTQ" %%mm4, 16%0" :"=m"(*d) :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh) :"memory"); d += 24; s += 8; } __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { register uint16_t bgr; bgr = *s++; *d++ = (bgr&0x1F)<<3; *d++ = (bgr&0x7E0)>>3; *d++ = (bgr&0xF800)>>8; } } /* * mm0 = 00 B3 00 B2 00 B1 00 B0 * mm1 = 00 G3 00 G2 00 G1 00 G0 * mm2 = 00 R3 00 R2 00 R1 00 R0 * mm6 = FF FF FF FF FF FF FF FF * mm7 = 00 00 00 00 00 00 00 00 */ #define PACK_RGB32 \ "packuswb %%mm7, %%mm0 \n\t" /* 00 00 00 00 B3 B2 B1 B0 */ \ "packuswb %%mm7, %%mm1 \n\t" /* 00 00 00 00 G3 G2 G1 G0 */ \ "packuswb %%mm7, %%mm2 \n\t" /* 00 00 00 00 R3 R2 R1 R0 */ \ "punpcklbw %%mm1, %%mm0 \n\t" /* G3 B3 G2 B2 G1 B1 G0 B0 */ \ "punpcklbw %%mm6, %%mm2 \n\t" /* FF R3 FF R2 FF R1 FF R0 */ \ "movq %%mm0, %%mm3 \n\t" \ "punpcklwd %%mm2, %%mm0 \n\t" /* FF R1 G1 B1 FF R0 G0 B0 */ \ "punpckhwd %%mm2, %%mm3 \n\t" /* FF R3 G3 B3 FF R2 G2 B2 */ \ MOVNTQ" %%mm0, %0 \n\t" \ MOVNTQ" %%mm3, 8%0 \n\t" \ static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, long src_size) { const uint16_t *end; #if HAVE_MMX const uint16_t *mm_end; #endif uint8_t *d = dst; const uint16_t *s = (const uint16_t *)src; end = s + src_size/2; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory"); __asm__ volatile("pxor %%mm7,%%mm7 \n\t":::"memory"); __asm__ volatile("pcmpeqd %%mm6,%%mm6 \n\t":::"memory"); mm_end = end - 3; while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movq %1, %%mm0 \n\t" "movq %1, %%mm1 \n\t" "movq %1, %%mm2 \n\t" "pand %2, %%mm0 \n\t" "pand %3, %%mm1 \n\t" "pand %4, %%mm2 \n\t" "psllq $3, %%mm0 \n\t" "psrlq $2, %%mm1 \n\t" "psrlq $7, %%mm2 \n\t" PACK_RGB32 :"=m"(*d) :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r) :"memory"); d += 16; s += 4; } __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { #if 0 //slightly slower on Athlon int bgr= *s++; *((uint32_t*)d)++ = ((bgr&0x1F)<<3) + ((bgr&0x3E0)<<6) + ((bgr&0x7C00)<<9); #else register uint16_t bgr; bgr = *s++; #ifdef WORDS_BIGENDIAN *d++ = 255; *d++ = (bgr&0x7C00)>>7; *d++ = (bgr&0x3E0)>>2; *d++ = (bgr&0x1F)<<3; #else *d++ = (bgr&0x1F)<<3; *d++ = (bgr&0x3E0)>>2; *d++ = (bgr&0x7C00)>>7; *d++ = 255; #endif #endif } } static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, long src_size) { const uint16_t *end; #if HAVE_MMX const uint16_t *mm_end; #endif uint8_t *d = dst; const uint16_t *s = (const uint16_t*)src; end = s + src_size/2; #if HAVE_MMX __asm__ volatile(PREFETCH" %0"::"m"(*s):"memory"); __asm__ volatile("pxor %%mm7,%%mm7 \n\t":::"memory"); __asm__ volatile("pcmpeqd %%mm6,%%mm6 \n\t":::"memory"); mm_end = end - 3; while (s < mm_end) { __asm__ volatile( PREFETCH" 32%1 \n\t" "movq %1, %%mm0 \n\t" "movq %1, %%mm1 \n\t" "movq %1, %%mm2 \n\t" "pand %2, %%mm0 \n\t" "pand %3, %%mm1 \n\t" "pand %4, %%mm2 \n\t" "psllq $3, %%mm0 \n\t" "psrlq $3, %%mm1 \n\t" "psrlq $8, %%mm2 \n\t" PACK_RGB32 :"=m"(*d) :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r) :"memory"); d += 16; s += 4; } __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); #endif while (s < end) { register uint16_t bgr; bgr = *s++; #ifdef WORDS_BIGENDIAN *d++ = 255; *d++ = (bgr&0xF800)>>8; *d++ = (bgr&0x7E0)>>3; *d++ = (bgr&0x1F)<<3; #else *d++ = (bgr&0x1F)<<3; *d++ = (bgr&0x7E0)>>3; *d++ = (bgr&0xF800)>>8; *d++ = 255; #endif } } static inline void RENAME(rgb32tobgr32)(const uint8_t *src, uint8_t *dst, long src_size) { #if HAVE_MMX x86_reg #else long #endif idx = 15 - src_size; const uint8_t *s = src-idx; uint8_t *d = dst-idx; #if HAVE_MMX __asm__ volatile( "test %0, %0 \n\t" "jns 2f \n\t" PREFETCH" (%1, %0) \n\t" "movq %3, %%mm7 \n\t" "pxor %4, %%mm7 \n\t" "movq %%mm7, %%mm6 \n\t" "pxor %5, %%mm7 \n\t" ASMALIGN(4) "1: \n\t" PREFETCH" 32(%1, %0) \n\t" "movq (%1, %0), %%mm0 \n\t" "movq 8(%1, %0), %%mm1 \n\t" # if HAVE_MMX2 "pshufw $177, %%mm0, %%mm3 \n\t" "pshufw $177, %%mm1, %%mm5 \n\t" "pand %%mm7, %%mm0 \n\t" "pand %%mm6, %%mm3 \n\t" "pand %%mm7, %%mm1 \n\t" "pand %%mm6, %%mm5 \n\t" "por %%mm3, %%mm0 \n\t" "por %%mm5, %%mm1 \n\t" # else "movq %%mm0, %%mm2 \n\t" "movq %%mm1, %%mm4 \n\t" "pand %%mm7, %%mm0 \n\t" "pand %%mm6, %%mm2 \n\t" "pand %%mm7, %%mm1 \n\t" "pand %%mm6, %%mm4 \n\t" "movq %%mm2, %%mm3 \n\t" "movq %%mm4, %%mm5 \n\t" "pslld $16, %%mm2 \n\t" "psrld $16, %%mm3 \n\t" "pslld $16, %%mm4 \n\t" "psrld $16, %%mm5 \n\t" "por %%mm2, %%mm0 \n\t" "por %%mm4, %%mm1 \n\t" "por %%mm3, %%mm0 \n\t" "por %%mm5, %%mm1 \n\t" # endif MOVNTQ" %%mm0, (%2, %0) \n\t" MOVNTQ" %%mm1, 8(%2, %0) \n\t" "add $16, %0 \n\t" "js 1b \n\t" SFENCE" \n\t" EMMS" \n\t" "2: \n\t" : "+&r"(idx) : "r" (s), "r" (d), "m" (mask32b), "m" (mask32r), "m" (mmx_one) : "memory"); #endif for (; idx<15; idx+=4) { register int v = *(const uint32_t *)&s[idx], g = v & 0xff00ff00; v &= 0xff00ff; *(uint32_t *)&d[idx] = (v>>16) + g + (v<<16); } } static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, long src_size) { unsigned i; #if HAVE_MMX x86_reg mmx_size= 23 - src_size; __asm__ volatile ( "test %%"REG_a", %%"REG_a" \n\t" "jns 2f \n\t" "movq "MANGLE(mask24r)", %%mm5 \n\t" "movq "MANGLE(mask24g)", %%mm6 \n\t" "movq "MANGLE(mask24b)", %%mm7 \n\t" ASMALIGN(4) "1: \n\t" PREFETCH" 32(%1, %%"REG_a") \n\t" "movq (%1, %%"REG_a"), %%mm0 \n\t" // BGR BGR BG "movq (%1, %%"REG_a"), %%mm1 \n\t" // BGR BGR BG "movq 2(%1, %%"REG_a"), %%mm2 \n\t" // R BGR BGR B "psllq $16, %%mm0 \n\t" // 00 BGR BGR "pand %%mm5, %%mm0 \n\t" "pand %%mm6, %%mm1 \n\t" "pand %%mm7, %%mm2 \n\t" "por %%mm0, %%mm1 \n\t" "por %%mm2, %%mm1 \n\t" "movq 6(%1, %%"REG_a"), %%mm0 \n\t" // BGR BGR BG MOVNTQ" %%mm1, (%2, %%"REG_a") \n\t" // RGB RGB RG "movq 8(%1, %%"REG_a"), %%mm1 \n\t" // R BGR BGR B "movq 10(%1, %%"REG_a"), %%mm2 \n\t" // GR BGR BGR "pand %%mm7, %%mm0 \n\t" "pand %%mm5, %%mm1 \n\t" "pand %%mm6, %%mm2 \n\t" "por %%mm0, %%mm1 \n\t" "por %%mm2, %%mm1 \n\t" "movq 14(%1, %%"REG_a"), %%mm0 \n\t" // R BGR BGR B MOVNTQ" %%mm1, 8(%2, %%"REG_a") \n\t" // B RGB RGB R "movq 16(%1, %%"REG_a"), %%mm1 \n\t" // GR BGR BGR "movq 18(%1, %%"REG_a"), %%mm2 \n\t" // BGR BGR BG "pand %%mm6, %%mm0 \n\t" "pand %%mm7, %%mm1 \n\t" "pand %%mm5, %%mm2 \n\t" "por %%mm0, %%mm1 \n\t" "por %%mm2, %%mm1 \n\t" MOVNTQ" %%mm1, 16(%2, %%"REG_a") \n\t" "add $24, %%"REG_a" \n\t" " js 1b \n\t" "2: \n\t" : "+a" (mmx_size) : "r" (src-mmx_size), "r"(dst-mmx_size) ); __asm__ volatile(SFENCE:::"memory"); __asm__ volatile(EMMS:::"memory"); if (mmx_size==23) return; //finished, was multiple of 8 src+= src_size; dst+= src_size; src_size= 23-mmx_size; src-= src_size; dst-= src_size; #endif for (i=0; i>1; for (y=0; y>1; for (y=0; yyuy2 #if HAVE_FAST_64BIT int i; uint64_t *ldst = (uint64_t *) dst; const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc; for (i = 0; i < chromWidth; i += 2){ uint64_t k, l; k = uc[0] + (yc[0] << 8) + (vc[0] << 16) + (yc[1] << 24); l = uc[1] + (yc[2] << 8) + (vc[1] << 16) + (yc[3] << 24); *ldst++ = k + (l << 32); yc += 4; uc += 2; vc += 2; } #else int i, *idst = (int32_t *) dst; const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc; for (i = 0; i < chromWidth; i++){ #ifdef WORDS_BIGENDIAN *idst++ = (uc[0] << 24)+ (yc[0] << 16) + (vc[0] << 8) + (yc[1] << 0); #else *idst++ = uc[0] + (yc[0] << 8) + (vc[0] << 16) + (yc[1] << 24); #endif yc += 2; uc++; vc++; } #endif #endif if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) { usrc += chromStride; vsrc += chromStride; } ysrc += lumStride; dst += dstStride; } #if HAVE_MMX __asm__( EMMS" \n\t" SFENCE" \n\t" :::"memory"); #endif } /** * Height should be a multiple of 2 and width should be a multiple of 16 * (If this is a problem for anyone then tell me, and I will fix it.) */ static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst, long width, long height, long lumStride, long chromStride, long dstStride) { //FIXME interpolate chroma RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2); } /** * Width should be a multiple of 16. */ static inline void RENAME(yuv422ptouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst, long width, long height, long lumStride, long chromStride, long dstStride) { RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1); } /** * Width should be a multiple of 16. */ static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst, long width, long height, long lumStride, long chromStride, long dstStride) { RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1); } /** * Height should be a multiple of 2 and width should be a multiple of 16. * (If this is a problem for anyone then tell me, and I will fix it.) */ static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst, long width, long height, long lumStride, long chromStride, long srcStride) { long y; const long chromWidth= width>>1; for (y=0; y>2; dst[2*x+2]= ( src[x] + 3*src[x+1])>>2; } dst[2*srcWidth-1]= src[srcWidth-1]; dst+= dstStride; for (y=1; y>2; dst[dstStride]= ( src[0] + 3*src[srcStride])>>2; for (x=mmxSize-1; x>2; dst[2*x+dstStride+2]= ( src[x+0] + 3*src[x+srcStride+1])>>2; dst[2*x+dstStride+1]= ( src[x+1] + 3*src[x+srcStride ])>>2; dst[2*x +2]= (3*src[x+1] + src[x+srcStride ])>>2; } dst[srcWidth*2 -1 ]= (3*src[srcWidth-1] + src[srcWidth-1 + srcStride])>>2; dst[srcWidth*2 -1 + dstStride]= ( src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2; dst+=dstStride*2; src+=srcStride; } // last line #if 1 dst[0]= src[0]; for (x=0; x>2; dst[2*x+2]= ( src[x] + 3*src[x+1])>>2; } dst[2*srcWidth-1]= src[srcWidth-1]; #else for (x=0; x>1; for (y=0; y>1; #if HAVE_MMX x86_reg cw = chromWidth; for (y=0; y>RGB2YUV_SHIFT) + 16; unsigned int V = ((RV*r + GV*g + BV*b)>>RGB2YUV_SHIFT) + 128; unsigned int U = ((RU*r + GU*g + BU*b)>>RGB2YUV_SHIFT) + 128; udst[i] = U; vdst[i] = V; ydst[2*i] = Y; b = src[6*i+3]; g = src[6*i+4]; r = src[6*i+5]; Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16; ydst[2*i+1] = Y; } ydst += lumStride; src += srcStride; for (i=0; i>RGB2YUV_SHIFT) + 16; ydst[2*i] = Y; b = src[6*i+3]; g = src[6*i+4]; r = src[6*i+5]; Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16; ydst[2*i+1] = Y; } udst += chromStride; vdst += chromStride; ydst += lumStride; src += srcStride; } } static void RENAME(interleaveBytes)(uint8_t *src1, uint8_t *src2, uint8_t *dest, long width, long height, long src1Stride, long src2Stride, long dstStride){ long h; for (h=0; h < height; h++) { long w; #if HAVE_MMX #if HAVE_SSE2 __asm__( "xor %%"REG_a", %%"REG_a" \n\t" "1: \n\t" PREFETCH" 64(%1, %%"REG_a") \n\t" PREFETCH" 64(%2, %%"REG_a") \n\t" "movdqa (%1, %%"REG_a"), %%xmm0 \n\t" "movdqa (%1, %%"REG_a"), %%xmm1 \n\t" "movdqa (%2, %%"REG_a"), %%xmm2 \n\t" "punpcklbw %%xmm2, %%xmm0 \n\t" "punpckhbw %%xmm2, %%xmm1 \n\t" "movntdq %%xmm0, (%0, %%"REG_a", 2) \n\t" "movntdq %%xmm1, 16(%0, %%"REG_a", 2) \n\t" "add $16, %%"REG_a" \n\t" "cmp %3, %%"REG_a" \n\t" " jb 1b \n\t" ::"r"(dest), "r"(src1), "r"(src2), "r" ((x86_reg)width-15) : "memory", "%"REG_a"" ); #else __asm__( "xor %%"REG_a", %%"REG_a" \n\t" "1: \n\t" PREFETCH" 64(%1, %%"REG_a") \n\t" PREFETCH" 64(%2, %%"REG_a") \n\t" "movq (%1, %%"REG_a"), %%mm0 \n\t" "movq 8(%1, %%"REG_a"), %%mm2 \n\t" "movq %%mm0, %%mm1 \n\t" "movq %%mm2, %%mm3 \n\t" "movq (%2, %%"REG_a"), %%mm4 \n\t" "movq 8(%2, %%"REG_a"), %%mm5 \n\t" "punpcklbw %%mm4, %%mm0 \n\t" "punpckhbw %%mm4, %%mm1 \n\t" "punpcklbw %%mm5, %%mm2 \n\t" "punpckhbw %%mm5, %%mm3 \n\t" MOVNTQ" %%mm0, (%0, %%"REG_a", 2) \n\t" MOVNTQ" %%mm1, 8(%0, %%"REG_a", 2) \n\t" MOVNTQ" %%mm2, 16(%0, %%"REG_a", 2) \n\t" MOVNTQ" %%mm3, 24(%0, %%"REG_a", 2) \n\t" "add $16, %%"REG_a" \n\t" "cmp %3, %%"REG_a" \n\t" " jb 1b \n\t" ::"r"(dest), "r"(src1), "r"(src2), "r" ((x86_reg)width-15) : "memory", "%"REG_a ); #endif for (w= (width&(~15)); w < width; w++) { dest[2*w+0] = src1[w]; dest[2*w+1] = src2[w]; } #else for (w=0; w < width; w++) { dest[2*w+0] = src1[w]; dest[2*w+1] = src2[w]; } #endif dest += dstStride; src1 += src1Stride; src2 += src2Stride; } #if HAVE_MMX __asm__( EMMS" \n\t" SFENCE" \n\t" ::: "memory" ); #endif } static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2, uint8_t *dst1, uint8_t *dst2, long width, long height, long srcStride1, long srcStride2, long dstStride1, long dstStride2) { long x,y,w,h; w=width/2; h=height/2; #if HAVE_MMX __asm__ volatile( PREFETCH" %0 \n\t" PREFETCH" %1 \n\t" ::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory"); #endif for (y=0;y>1); uint8_t* d=dst1+dstStride1*y; x=0; #if HAVE_MMX for (;x>1); uint8_t* d=dst2+dstStride2*y; x=0; #if HAVE_MMX for (;x>2); const uint8_t* vp=src3+srcStride3*(y>>2); uint8_t* d=dst+dstStride*y; #if HAVE_MMX x86_reg x = 0; for (;x