ref: bd8d97dddbb2a86e506d060788cd662ded2a2aa4
dir: /codec/encoder/core/x86/matrix_transpose.asm/
;*! ;* \copy ;* Copyright (c) 2009-2013, Cisco Systems ;* All rights reserved. ;* ;* Redistribution and use in source and binary forms, with or without ;* modification, are permitted provided that the following conditions ;* are met: ;* ;* ?Redistributions of source code must retain the above copyright ;* notice, this list of conditions and the following disclaimer. ;* ;* ?Redistributions in binary form must reproduce the above copyright ;* notice, this list of conditions and the following disclaimer in ;* the documentation and/or other materials provided with the ;* distribution. ;* ;* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ;* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ;* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS ;* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ;* COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, ;* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, ;* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ;* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER ;* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT ;* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ;* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE ;* POSSIBILITY OF SUCH DAMAGE. ;*************************************************************************/ %include "asm_inc.asm" ;in: m0, m1, m2, m3, m4, m5, m6, m7 ;out: m0, m3, m5, m2, m7, m1, m6, m4 %macro TRANSPOSE_8x8B_MMX 10 MMX_XSwap bw, %1, %2, %8 MMX_XSwap bw, %3, %4, %2 MMX_XSwap bw, %5, %6, %4 movq %6, %9 movq %10, %4 MMX_XSwap bw, %7, %6, %4 MMX_XSwap wd, %1, %3, %6 MMX_XSwap wd, %8, %2, %3 MMX_XSwap wd, %5, %7, %2 movq %7, %10 movq %10, %3 MMX_XSwap wd, %7, %4, %3 MMX_XSwap dq, %1, %5, %4 MMX_XSwap dq, %6, %2, %5 MMX_XSwap dq, %8, %7, %2 movq %7, %10 movq %10, %5 MMX_XSwap dq, %7, %3, %5 movq %3, %10 %endmacro ;in: m0, m3, m5, m2, m7, m1, m6, m4 %macro TRANSPOSE8x8_WRITE_MMX 2 ; dst, dst_stride movq [%1], mm0 ; result of line 1, x8 bytes movq [%1+%2], mm3 ; result of line 2 lea %1, [%1+2*%2] movq [%1], mm5 ; result of line 3 movq [%1+%2], mm2 ; result of line 4 lea %1, [%1+2*%2] movq [%1], mm7 ; result of line 5 movq [%1+%2], mm1 ; result of line 6 lea %1, [%1+2*%2] movq [%1], mm6 ; result of line 7 movq [%1+%2], mm4 ; result of line 8 %endmacro ;in: m0, m3, m5, m2, m7, m1, m6, m4 %macro TRANSPOSE8x8_WRITE_ALT_MMX 3 ; dst, dst_stride, reg32 movq [%1], mm0 ; result of line 1, x8 bytes movq [%1+%2], mm3 ; result of line 2 lea %3, [%1+2*%2] movq [%3], mm5 ; result of line 3 movq [%3+%2], mm2 ; result of line 4 lea %3, [%3+2*%2] movq [%3], mm7 ; result of line 5 movq [%3+%2], mm1 ; result of line 6 lea %3, [%3+2*%2] movq [%3], mm6 ; result of line 7 movq [%3+%2], mm4 ; result of line 8 %endmacro ; end of TRANSPOSE8x8_WRITE_ALT_MMX ; for transpose 16x8 ;in: m0, m1, m2, m3, m4, m5, m6, m7 ;out: m4, m2, m3, m7, m5, m1, m6, m0 %macro TRANSPOSE_8x16B_SSE2 10 SSE2_XSawp bw, %1, %2, %8 SSE2_XSawp bw, %3, %4, %2 SSE2_XSawp bw, %5, %6, %4 movdqa %6, %9 movdqa %10, %4 SSE2_XSawp bw, %7, %6, %4 SSE2_XSawp wd, %1, %3, %6 SSE2_XSawp wd, %8, %2, %3 SSE2_XSawp wd, %5, %7, %2 movdqa %7, %10 movdqa %10, %3 SSE2_XSawp wd, %7, %4, %3 SSE2_XSawp dq, %1, %5, %4 SSE2_XSawp dq, %6, %2, %5 SSE2_XSawp dq, %8, %7, %2 movdqa %7, %10 movdqa %10, %5 SSE2_XSawp dq, %7, %3, %5 SSE2_XSawp qdq, %1, %8, %3 SSE2_XSawp qdq, %4, %2, %8 SSE2_XSawp qdq, %6, %7, %2 movdqa %7, %10 movdqa %10, %1 SSE2_XSawp qdq, %7, %5, %1 movdqa %5, %10 %endmacro ; end of TRANSPOSE_8x16B_SSE2 %macro TRANSPOSE8x16_WRITE_SSE2 2 ; dst, dst_stride movq [%1], xmm4 ; result of line 1, x8 bytes movq [%1+%2], xmm2 ; result of line 2 lea %1, [%1+2*%2] movq [%1], xmm3 ; result of line 3 movq [%1+%2], xmm7 ; result of line 4 lea %1, [%1+2*%2] movq [%1], xmm5 ; result of line 5 movq [%1+%2], xmm1 ; result of line 6 lea %1, [%1+2*%2] movq [%1], xmm6 ; result of line 7 movq [%1+%2], xmm0 ; result of line 8 lea %1, [%1+2*%2] movhpd [%1], xmm4 ; result of line 9 movhpd [%1+%2], xmm2 ; result of line 10 lea %1, [%1+2*%2] movhpd [%1], xmm3 ; result of line 11 movhpd [%1+%2], xmm7 ; result of line 12 lea %1, [%1+2*%2] movhpd [%1], xmm5 ; result of line 13 movhpd [%1+%2], xmm1 ; result of line 14 lea %1, [%1+2*%2] movhpd [%1], xmm6 ; result of line 15 movhpd [%1+%2], xmm0 ; result of line 16 %endmacro ; end of TRANSPOSE_WRITE_RESULT_SSE2 %macro TRANSPOSE8x16_WRITE_ALT_SSE2 3 ; dst, dst_stride, reg32 movq [%1], xmm4 ; result of line 1, x8 bytes movq [%1+%2], xmm2 ; result of line 2 lea %3, [%1+2*%2] movq [%3], xmm3 ; result of line 3 movq [%3+%2], xmm7 ; result of line 4 lea %3, [%3+2*%2] movq [%3], xmm5 ; result of line 5 movq [%3+%2], xmm1 ; result of line 6 lea %3, [%3+2*%2] movq [%3], xmm6 ; result of line 7 movq [%3+%2], xmm0 ; result of line 8 lea %3, [%3+2*%2] movhpd [%3], xmm4 ; result of line 9 movhpd [%3+%2], xmm2 ; result of line 10 lea %3, [%3+2*%2] movhpd [%3], xmm3 ; result of line 11 movhpd [%3+%2], xmm7 ; result of line 12 lea %3, [%3+2*%2] movhpd [%3], xmm5 ; result of line 13 movhpd [%3+%2], xmm1 ; result of line 14 lea %3, [%3+2*%2] movhpd [%3], xmm6 ; result of line 15 movhpd [%3+%2], xmm0 ; result of line 16 %endmacro ; end of TRANSPOSE8x16_WRITE_ALT_SSE2 SECTION .text WELS_EXTERN TransposeMatrixBlock16x16_sse2 ; void TransposeMatrixBlock16x16_sse2( void *dst/*16x16*/, const int32_t dst_stride, void *src/*16x16*/, const int32_t src_stride ); push r4 push r5 %assign push_num 2 LOAD_4_PARA PUSH_XMM 8 SIGN_EXTENSION r1, r1d SIGN_EXTENSION r3, r3d mov r4, r7 and r4, 0Fh sub r7, 10h sub r7, r4 lea r5, [r3+r3*2] ; top 8x16 block movdqa xmm0, [r2] movdqa xmm1, [r2+r3] movdqa xmm2, [r2+r3*2] movdqa xmm3, [r2+r5] lea r2, [r2+r3*4] movdqa xmm4, [r2] movdqa xmm5, [r2+r3] movdqa xmm6, [r2+r3*2] ;in: m0, m1, m2, m3, m4, m5, m6, m7 ;out: m4, m2, m3, m7, m5, m1, m6, m0 TRANSPOSE_8x16B_SSE2 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, [r2+r5], [r7] TRANSPOSE8x16_WRITE_SSE2 r0, r1 ; bottom 8x16 block lea r2, [r2+r3*4] movdqa xmm0, [r2] movdqa xmm1, [r2+r3] movdqa xmm2, [r2+r3*2] movdqa xmm3, [r2+r5] lea r2, [r2+r3*4] movdqa xmm4, [r2] movdqa xmm5, [r2+r3] movdqa xmm6, [r2+r3*2] ;in: m0, m1, m2, m3, m4, m5, m6, m7 ;out: m4, m2, m3, m7, m5, m1, m6, m0 TRANSPOSE_8x16B_SSE2 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, [r2+r5], [r7] mov r5, r1 sal r5, 4 sub r0, r5 lea r0, [r0+r1*2+8] TRANSPOSE8x16_WRITE_SSE2 r0, r1 add r7, r4 add r7, 10h POP_XMM LOAD_4_PARA_POP pop r5 pop r4 ret WELS_EXTERN TransposeMatrixBlocksx16_sse2 ; void TransposeMatrixBlocksx16_sse2( void *dst/*W16x16*/, const int32_t dst_stride, void *src/*16xW16*/, const int32_t src_stride, const int32_t num_blocks ); push r5 push r6 %assign push_num 2 LOAD_5_PARA PUSH_XMM 8 SIGN_EXTENSION r1, r1d SIGN_EXTENSION r3, r3d SIGN_EXTENSION r4, r4d mov r5, r7 and r5, 0Fh sub r7, 10h sub r7, r5 TRANSPOSE_LOOP_SSE2: ; explictly loading next loop data lea r6, [r2+r3*8] push r4 %rep 8 mov r4, [r6] mov r4, [r6+r3] lea r6, [r6+r3*2] %endrep pop r4 ; top 8x16 block movdqa xmm0, [r2] movdqa xmm1, [r2+r3] lea r2, [r2+r3*2] movdqa xmm2, [r2] movdqa xmm3, [r2+r3] lea r2, [r2+r3*2] movdqa xmm4, [r2] movdqa xmm5, [r2+r3] lea r2, [r2+r3*2] movdqa xmm6, [r2] ;in: m0, m1, m2, m3, m4, m5, m6, m7 ;out: m4, m2, m3, m7, m5, m1, m6, m0 TRANSPOSE_8x16B_SSE2 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, [r2+r3], [r7] TRANSPOSE8x16_WRITE_ALT_SSE2 r0, r1, r6 lea r2, [r2+r3*2] ; bottom 8x16 block movdqa xmm0, [r2] movdqa xmm1, [r2+r3] lea r2, [r2+r3*2] movdqa xmm2, [r2] movdqa xmm3, [r2+r3] lea r2, [r2+r3*2] movdqa xmm4, [r2] movdqa xmm5, [r2+r3] lea r2, [r2+r3*2] movdqa xmm6, [r2] ;in: m0, m1, m2, m3, m4, m5, m6, m7 ;out: m4, m2, m3, m7, m5, m1, m6, m0 TRANSPOSE_8x16B_SSE2 xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, [r2+r3], [r7] TRANSPOSE8x16_WRITE_ALT_SSE2 r0+8, r1, r6 lea r2, [r2+r3*2] lea r0, [r0+16] dec r4 jg near TRANSPOSE_LOOP_SSE2 add r7, r5 add r7, 10h POP_XMM LOAD_5_PARA_POP pop r6 pop r5 ret WELS_EXTERN TransposeMatrixBlock8x8_mmx ; void TransposeMatrixBlock8x8_mmx( void *dst/*8x8*/, const int32_t dst_stride, void *src/*8x8*/, const int32_t src_stride ); %assign push_num 0 LOAD_4_PARA SIGN_EXTENSION r1, r1d SIGN_EXTENSION r3, r3d sub r7, 8 movq mm0, [r2] movq mm1, [r2+r3] lea r2, [r2+2*r3] movq mm2, [r2] movq mm3, [r2+r3] lea r2, [r2+2*r3] movq mm4, [r2] movq mm5, [r2+r3] lea r2, [r2+2*r3] movq mm6, [r2] ;in: m0, m1, m2, m3, m4, m5, m6, m7 ;out: m0, m3, m5, m2, m7, m1, m6, m4 TRANSPOSE_8x8B_MMX mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, [r2+r3], [r7] TRANSPOSE8x8_WRITE_MMX r0, r1 emms add r7, 8 LOAD_4_PARA_POP ret WELS_EXTERN TransposeMatrixBlocksx8_mmx ; void TransposeMatrixBlocksx8_mmx( void *dst/*8xW8*/, const int32_t dst_stride, void *src/*W8x8*/, const int32_t src_stride, const int32_t num_blocks ); push r5 push r6 %assign push_num 2 LOAD_5_PARA SIGN_EXTENSION r1, r1d SIGN_EXTENSION r3, r3d SIGN_EXTENSION r4, r4d sub r7, 8 lea r5, [r2+r3*8] TRANSPOSE_BLOCKS_X8_LOOP_MMX: ; explictly loading next loop data %rep 4 mov r6, [r5] mov r6, [r5+r3] lea r5, [r5+r3*2] %endrep movq mm0, [r2] movq mm1, [r2+r3] lea r2, [r2+2*r3] movq mm2, [r2] movq mm3, [r2+r3] lea r2, [r2+2*r3] movq mm4, [r2] movq mm5, [r2+r3] lea r2, [r2+2*r3] movq mm6, [r2] ;in: m0, m1, m2, m3, m4, m5, m6, m7 ;out: m0, m3, m5, m2, m7, m1, m6, m4 TRANSPOSE_8x8B_MMX mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, [r2+r3], [r7] TRANSPOSE8x8_WRITE_ALT_MMX r0, r1, r6 lea r0, [r0+8] lea r2, [r2+2*r3] dec r4 jg near TRANSPOSE_BLOCKS_X8_LOOP_MMX emms add r7, 8 LOAD_5_PARA_POP pop r6 pop r5 ret