ref: 5fa6c44a61fbf946646899d9db24d92cdce478ac
dir: /src/x86/itx_ssse3.asm/
; Copyright © 2018, VideoLAN and dav1d authors ; Copyright © 2018, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: ; ; 1. Redistributions of source code must retain the above copyright notice, this ; list of conditions and the following disclaimer. ; ; 2. Redistributions in binary form must reproduce the above copyright notice, ; this list of conditions and the following disclaimer in the documentation ; and/or other materials provided with the distribution. ; ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %include "config.asm" %include "ext/x86/x86inc.asm" SECTION_RODATA 16 deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 deint_shuf1: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 deint_shuf2: db 8, 9, 0, 1, 10, 11, 2, 3, 12, 13, 4, 5, 14, 15, 6, 7 %macro COEF_PAIR 2 pw_%1_m%2: times 4 dw %1, -%2 pw_%2_%1: times 4 dw %2, %1 %endmacro ;adst4 pw_1321_3803: times 4 dw 1321, 3803 pw_2482_m1321: times 4 dw 2482, -1321 pw_3344_2482: times 4 dw 3344, 2482 pw_3344_m3803: times 4 dw 3344, -3803 pw_m6688_m3803: times 4 dw -6688, -3803 COEF_PAIR 1567, 3784 COEF_PAIR 799, 4017 COEF_PAIR 3406, 2276 COEF_PAIR 401, 4076 COEF_PAIR 1931, 3612 COEF_PAIR 3166, 2598 COEF_PAIR 3920, 1189 COEF_PAIR 3784, 1567 pd_2048: times 4 dd 2048 pw_2048: times 8 dw 2048 pw_4096: times 8 dw 4096 pw_16384: times 8 dw 16384 pw_m16384: times 8 dw -16384 pw_2896x8: times 8 dw 2896*8 pw_3344x8: times 8 dw 3344*8 pw_5793x4: times 8 dw 5793*4 iadst4_dconly1a: times 2 dw 10568, 19856, 26752, 30424 iadst4_dconly1b: times 2 dw 30424, 26752, 19856, 10568 iadst4_dconly2a: dw 10568, 10568, 10568, 10568, 19856, 19856, 19856, 19856 iadst4_dconly2b: dw 26752, 26752, 26752, 26752, 30424, 30424, 30424, 30424 SECTION .text %macro REPX 2-* %xdefine %%f(x) %1 %rep %0 - 1 %rotate 1 %%f(%1) %endrep %endmacro %define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) %if ARCH_X86_64 %define o(x) x %else %define o(x) r5-$$+x ; PIC %endif %macro WRITE_4X4 9 ;src[1-2], tmp[1-3], row[1-4] lea r2, [dstq+strideq*2] %assign %%i 1 %rotate 5 %rep 4 %if %1 & 2 CAT_XDEFINE %%row_adr, %%i, r2 + strideq*(%1&1) %else CAT_XDEFINE %%row_adr, %%i, dstq + strideq*(%1&1) %endif %assign %%i %%i + 1 %rotate 1 %endrep movd m%3, [%%row_adr1] ;dst0 movd m%5, [%%row_adr2] ;dst1 punpckldq m%3, m%5 ;high: dst1 :low: dst0 movd m%4, [%%row_adr3] ;dst2 movd m%5, [%%row_adr4] ;dst3 punpckldq m%4, m%5 ;high: dst3 :low: dst2 pxor m%5, m%5 punpcklbw m%3, m%5 ;extend byte to word punpcklbw m%4, m%5 ;extend byte to word paddw m%1, m%3 ;high: dst1 + out1 ;low: dst0 + out0 paddw m%2, m%4 ;high: dst3 + out3 ;low: dst2 + out2 packuswb m%1, m%2 ;high->low: dst3 + out3, dst2 + out2, dst1 + out1, dst0 + out0 movd [%%row_adr1], m%1 ;store dst0 + out0 pshuflw m%2, m%1, q1032 movd [%%row_adr2], m%2 ;store dst1 + out1 punpckhqdq m%1, m%1 movd [%%row_adr3], m%1 ;store dst2 + out2 psrlq m%1, 32 movd [%%row_adr4], m%1 ;store dst3 + out3 %endmacro %macro ITX4_END 4-5 2048 ; row[1-4], rnd %if %5 mova m2, [o(pw_%5)] pmulhrsw m0, m2 pmulhrsw m1, m2 %endif WRITE_4X4 0, 1, 2, 3, 4, %1, %2, %3, %4 ret %endmacro ; flags: 1 = swap, 2: coef_regs %macro ITX_MUL2X_PACK 5-6 0 ; dst/src, tmp[1], rnd, coef[1-2], flags %if %6 & 2 pmaddwd m%2, m%4, m%1 pmaddwd m%1, m%5 %elif %6 & 1 pmaddwd m%2, m%1, [o(pw_%5_%4)] pmaddwd m%1, [o(pw_%4_m%5)] %else pmaddwd m%2, m%1, [o(pw_%4_m%5)] pmaddwd m%1, [o(pw_%5_%4)] %endif paddd m%2, m%3 paddd m%1, m%3 psrad m%2, 12 psrad m%1, 12 packssdw m%1, m%2 %endmacro %macro IDCT4_1D_PACKED 0-1 ;pw_2896x8 punpckhwd m2, m0, m1 ;unpacked in1 in3 psubw m3, m0, m1 paddw m0, m1 punpcklqdq m0, m3 ;high: in0-in2 ;low: in0+in2 mova m3, [o(pd_2048)] ITX_MUL2X_PACK 2, 1, 3, 1567, 3784 %if %0 == 1 pmulhrsw m0, m%1 %else pmulhrsw m0, [o(pw_2896x8)] ;high: t1 ;low: t0 %endif psubsw m1, m0, m2 ;high: out2 ;low: out3 paddsw m0, m2 ;high: out1 ;low: out0 %endmacro %macro IADST4_1D_PACKED 0 punpcklwd m2, m0, m1 ;unpacked in0 in2 punpckhwd m3, m0, m1 ;unpacked in1 in3 psubw m0, m1 punpckhqdq m1, m1 ; paddw m1, m0 ;low: in0 - in2 + in3 pmaddwd m0, m2, [o(pw_1321_3803)] ;1321 * in0 + 3803 * in2 pmaddwd m2, [o(pw_2482_m1321)] ;2482 * in0 - 1321 * in2 pmaddwd m4, m3, [o(pw_3344_2482)] ;3344 * in1 + 2482 * in3 pmaddwd m5, m3, [o(pw_3344_m3803)];3344 * in1 - 3803 * in3 paddd m4, m0 ;t0 + t3 pmaddwd m3, [o(pw_m6688_m3803)] ;-2 * 3344 * in1 - 3803 * in3 pmulhrsw m1, [o(pw_3344x8)] ;low: out2 mova m0, [o(pd_2048)] paddd m2, m0 paddd m0, m4 ;t0 + t3 + 2048 paddd m5, m2 ;t1 + t3 + 2048 paddd m2, m4 paddd m2, m3 ;t0 + t1 - t3 + 2048 psrad m0, 12 ;out0 psrad m5, 12 ;out1 psrad m2, 12 ;out3 packssdw m0, m5 ;high: out1 ;low: out0 packssdw m2, m2 ;high: out3 ;low: out3 %endmacro %macro INV_TXFM_FN 5+ ; type1, type2, fast_thresh, size, xmm/stack cglobal inv_txfm_add_%1_%2_%4, 4, 6, %5, dst, stride, coeff, eob, tx2 %undef cmp %define %%p1 m(i%1_%4_internal) %if ARCH_X86_32 LEA r5, $$ %endif %if has_epilogue %if %3 > 0 cmp eobd, %3 jle %%end %elif %3 == 0 test eobd, eobd jz %%end %endif lea tx2q, [o(m(i%2_%4_internal).pass2)] call %%p1 RET %%end: %else lea tx2q, [o(m(i%2_%4_internal).pass2)] %if %3 > 0 cmp eobd, %3 jg %%p1 %elif %3 == 0 test eobd, eobd jnz %%p1 %else times ((%%end - %%p1) >> 31) & 1 jmp %%p1 ALIGN function_align %%end: %endif %endif %endmacro %macro INV_TXFM_4X4_FN 2-3 -1 ; type1, type2, fast_thresh INV_TXFM_FN %1, %2, %3, 4x4, 6 %ifidn %1_%2, dct_identity mova m0, [o(pw_2896x8)] pmulhrsw m0, [coeffq] paddw m0, m0 pmulhrsw m0, [o(pw_5793x4)] punpcklwd m0, m0 punpckhdq m1, m0, m0 punpckldq m0, m0 TAIL_CALL m(iadst_4x4_internal).end %elifidn %1_%2, identity_dct mova m1, [coeffq+16*0] mova m2, [coeffq+16*1] punpcklwd m0, m1, m2 punpckhwd m1, m2 punpcklwd m0, m1 punpcklqdq m0, m0 paddw m0, m0 pmulhrsw m0, [o(pw_5793x4)] pmulhrsw m0, [o(pw_2896x8)] mova m1, m0 TAIL_CALL m(iadst_4x4_internal).end %elif %3 >= 0 pshuflw m0, [coeffq], q0000 punpcklqdq m0, m0 %ifidn %1, dct mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 %elifidn %1, adst pmulhrsw m0, [o(iadst4_dconly1a)] %elifidn %1, flipadst pmulhrsw m0, [o(iadst4_dconly1b)] %endif mov [coeffq], eobd ;0 %ifidn %2, dct %ifnidn %1, dct pmulhrsw m0, [o(pw_2896x8)] %else pmulhrsw m0, m1 %endif mova m1, m0 TAIL_CALL m(iadst_4x4_internal).end2 %else ; adst / flipadst pmulhrsw m1, m0, [o(iadst4_dconly2b)] pmulhrsw m0, [o(iadst4_dconly2a)] TAIL_CALL m(i%2_4x4_internal).end2 %endif %endif %endmacro INIT_XMM ssse3 INV_TXFM_4X4_FN dct, dct, 0 INV_TXFM_4X4_FN dct, adst, 0 INV_TXFM_4X4_FN dct, flipadst, 0 INV_TXFM_4X4_FN dct, identity, 3 cglobal idct_4x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*0] ;high: in1 ;low: in0 mova m1, [coeffq+16*1] ;high: in3 ;low in2 IDCT4_1D_PACKED mova m2, [o(deint_shuf)] shufps m3, m0, m1, q1331 shufps m0, m1, q0220 pshufb m0, m2 ;high: in1 ;low: in0 pshufb m1, m3, m2 ;high: in3 ;low :in2 jmp tx2q .pass2: IDCT4_1D_PACKED pxor m2, m2 mova [coeffq+16*0], m2 mova [coeffq+16*1], m2 ;memset(coeff, 0, sizeof(*coeff) * sh * sw); ITX4_END 0, 1, 3, 2 INV_TXFM_4X4_FN adst, dct, 0 INV_TXFM_4X4_FN adst, adst, 0 INV_TXFM_4X4_FN adst, flipadst, 0 INV_TXFM_4X4_FN adst, identity cglobal iadst_4x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] call .main punpckhwd m3, m0, m2 punpcklwd m0, m1 punpckhwd m1, m0, m3 ;high: in3 ;low :in2 punpcklwd m0, m3 ;high: in1 ;low: in0 jmp tx2q .pass2: call .main punpcklqdq m1, m2 ;out2 out3 .end: pxor m2, m2 mova [coeffq+16*0], m2 mova [coeffq+16*1], m2 .end2: ITX4_END 0, 1, 2, 3 ALIGN function_align .main: IADST4_1D_PACKED ret INV_TXFM_4X4_FN flipadst, dct, 0 INV_TXFM_4X4_FN flipadst, adst, 0 INV_TXFM_4X4_FN flipadst, flipadst, 0 INV_TXFM_4X4_FN flipadst, identity cglobal iflipadst_4x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] call m(iadst_4x4_internal).main punpcklwd m1, m0 punpckhwd m2, m0 punpcklwd m0, m2, m1 ;high: in3 ;low :in2 punpckhwd m2, m1 ;high: in1 ;low: in0 mova m1, m2 jmp tx2q .pass2: call m(iadst_4x4_internal).main punpcklqdq m1, m2 ;out2 out3 .end: pxor m2, m2 mova [coeffq+16*0], m2 mova [coeffq+16*1], m2 .end2: ITX4_END 3, 2, 1, 0 INV_TXFM_4X4_FN identity, dct, 3 INV_TXFM_4X4_FN identity, adst INV_TXFM_4X4_FN identity, flipadst INV_TXFM_4X4_FN identity, identity cglobal iidentity_4x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] mova m2, [o(pw_5793x4)] paddw m0, m0 paddw m1, m1 pmulhrsw m0, m2 pmulhrsw m1, m2 punpckhwd m2, m0, m1 punpcklwd m0, m1 punpckhwd m1, m0, m2 ;high: in3 ;low :in2 punpcklwd m0, m2 ;high: in1 ;low: in0 jmp tx2q .pass2: mova m2, [o(pw_5793x4)] paddw m0, m0 paddw m1, m1 pmulhrsw m0, m2 pmulhrsw m1, m2 jmp m(iadst_4x4_internal).end %macro IWHT4_1D_PACKED 0 punpckhqdq m3, m0, m1 ;low: in1 high: in3 punpcklqdq m0, m1 ;low: in0 high: in2 psubw m2, m0, m3 ;low: in0 - in1 high: in2 - in3 paddw m0, m3 ;low: in0 + in1 high: in2 + in3 punpckhqdq m2, m2 ;t2 t2 punpcklqdq m0, m0 ;t0 t0 psubw m1, m0, m2 psraw m1, 1 ;t4 t4 psubw m1, m3 ;low: t1/out2 high: t3/out1 psubw m0, m1 ;high: out0 paddw m2, m1 ;low: out3 %endmacro cglobal inv_txfm_add_wht_wht_4x4, 3, 3, 4, dst, stride, coeff mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] pxor m2, m2 mova [coeffq+16*0], m2 mova [coeffq+16*1], m2 psraw m0, 2 psraw m1, 2 IWHT4_1D_PACKED punpckhwd m0, m1 punpcklwd m3, m1, m2 punpckhdq m1, m0, m3 punpckldq m0, m3 IWHT4_1D_PACKED shufpd m0, m2, 0x01 ITX4_END 0, 3, 2, 1, 0 %macro IDCT8_1D_PACKED 0 mova m6, [o(pd_2048)] punpckhwd m5, m0, m3 ;unpacked in1 in7 punpckhwd m4, m2, m1 ;unpacked in5 in3 punpcklwd m1, m3 ;unpacked in2 in6 psubw m3, m0, m2 paddw m0, m2 punpcklqdq m0, m3 ;low: in0+in4 high: in0-in4 ITX_MUL2X_PACK 5, 2, 6, 799, 4017, 1 ;low: t4a high: t7a ITX_MUL2X_PACK 4, 2, 6, 3406, 2276, 1 ;low: t5a high: t6a ITX_MUL2X_PACK 1, 2, 6, 1567, 3784 ;low: t3 high: t2 mova m6, [o(pw_2896x8)] psubsw m2, m5, m4 ;low: t5a high: t6a paddsw m5, m4 ;low: t4 high: t7 punpckhqdq m4, m2, m2 ;low: t6a high: t6a psubw m3, m4, m2 ;low: t6a - t5a paddw m4, m2 ;low: t6a + t5a punpcklqdq m4, m3 ;low: t6a + t5a high: t6a - t5a pmulhrsw m0, m6 ;low: t0 high: t1 pmulhrsw m4, m6 ;low: t6 high: t5 shufps m2, m5, m4, q1032 ;low: t7 high: t6 shufps m5, m4, q3210 ;low: t4 high: t5 psubsw m4, m0, m1 ;low: tmp3 high: tmp2 paddsw m0, m1 ;low: tmp0 high: tmp1 psubsw m3, m0, m2 ;low: out7 high: out6 paddsw m0, m2 ;low: out0 high: out1 psubsw m2, m4, m5 ;low: out4 high: out5 paddsw m1, m4, m5 ;low: out3 high: out2 %endmacro ;dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 ;dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 %macro ITX_MULSUB_2W 7 ; dst/src[1-2], tmp[1-2], rnd, coef[1-2] punpckhwd m%3, m%1, m%2 punpcklwd m%1, m%2 %if %7 < 8 pmaddwd m%2, m%7, m%1 pmaddwd m%4, m%7, m%3 %else mova m%2, [o(pw_%7_%6)] pmaddwd m%4, m%3, m%2 pmaddwd m%2, m%1 %endif paddd m%4, m%5 paddd m%2, m%5 psrad m%4, 12 psrad m%2, 12 packssdw m%2, m%4 ;dst2 %if %7 < 8 pmaddwd m%3, m%6 pmaddwd m%1, m%6 %else mova m%4, [o(pw_%6_m%7)] pmaddwd m%3, m%4 pmaddwd m%1, m%4 %endif paddd m%3, m%5 paddd m%1, m%5 psrad m%3, 12 psrad m%1, 12 packssdw m%1, m%3 ;dst1 %endmacro %macro IDCT4_1D 7 ; src[1-4], tmp[1-2], pd_2048 ITX_MULSUB_2W %2, %4, %5, %6, %7, 1567, 3784 ;t2, t3 mova m%6, [o(pw_2896x8)] paddw m%5, m%1, m%3 psubw m%1, m%3 pmulhrsw m%1, m%6 ;t1 pmulhrsw m%5, m%6 ;t0 psubsw m%3, m%1, m%2 ;out2 paddsw m%2, m%1 ;out1 paddsw m%1, m%5, m%4 ;out0 psubsw m%5, m%4 ;out3 mova m%4, m%5 %endmacro %macro IADST4_1D 0 mova m4, m2 psubw m2, m0, m4 paddw m2, m3 ;low: in0 - in2 + in3 punpckhwd m6, m0, m4 ;unpacked in0 in2 punpckhwd m7, m1, m3 ;unpacked in1 in3 punpcklwd m0, m4 ;unpacked in0 in2 punpcklwd m1, m3 ;unpacked in1 in3 pmaddwd m4, m0, [o(pw_1321_3803)] ;1321 * in0 + 3803 * in2 pmaddwd m0, [o(pw_2482_m1321)] ;2482 * in0 - 1321 * in2 pmaddwd m3, m1, [o(pw_3344_2482)] ;3344 * in1 + 2482 * in3 pmaddwd m5, m1, [o(pw_3344_m3803)] ;3344 * in1 - 3803 * in3 paddd m3, m4 ;t0 + t3 pmaddwd m1, [o(pw_m6688_m3803)] ;-2 * 3344 * in1 - 3803 * in3 pmulhrsw m2, [o(pw_3344x8)] ;out2 mova m4, [o(pd_2048)] paddd m0, m4 paddd m4, m3 ;t0 + t3 + 2048 paddd m5, m0 ;t1 + t3 + 2048 paddd m3, m0 paddd m3, m1 ;t0 + t1 - t3 + 2048 psrad m4, 12 ;out0 psrad m5, 12 ;out1 psrad m3, 12 ;out3 packssdw m0, m4, m5 ;low: out0 high: out1 pmaddwd m4, m6, [o(pw_1321_3803)] ;1321 * in0 + 3803 * in2 pmaddwd m6, [o(pw_2482_m1321)] ;2482 * in0 - 1321 * in2 pmaddwd m1, m7, [o(pw_3344_2482)] ;3344 * in1 + 2482 * in3 pmaddwd m5, m7, [o(pw_3344_m3803)] ;3344 * in1 - 3803 * in3 paddd m1, m4 ;t0 + t3 pmaddwd m7, [o(pw_m6688_m3803)] ;-2 * 3344 * in1 - 3803 * in3 mova m4, [o(pd_2048)] paddd m6, m4 paddd m4, m1 ;t0 + t3 + 2048 paddd m5, m6 ;t1 + t3 + 2048 paddd m1, m6 paddd m1, m7 ;t0 + t1 - t3 + 2048 psrad m4, 12 ;out0 psrad m5, 12 ;out1 psrad m1, 12 ;out3 packssdw m3, m1 ;out3 packssdw m4, m5 ;low: out0 high: out1 punpckhqdq m1, m0, m4 ;out1 punpcklqdq m0, m4 ;out0 %endmacro %macro IADST8_1D_PACKED 0 mova m6, [o(pd_2048)] punpckhwd m4, m3, m0 ;unpacked in7 in0 punpckhwd m5, m2, m1 ;unpacked in5 in2 punpcklwd m1, m2 ;unpacked in3 in4 punpcklwd m0, m3 ;unpacked in1 in6 ITX_MUL2X_PACK 4, 2, 6, 401, 4076 ;low: t0a high: t1a ITX_MUL2X_PACK 5, 2, 6, 1931, 3612 ;low: t2a high: t3a ITX_MUL2X_PACK 1, 2, 6, 3166, 2598 ;low: t4a high: t5a ITX_MUL2X_PACK 0, 2, 6, 3920, 1189 ;low: t6a high: t7a psubsw m3, m4, m1 ;low: t4 high: t5 paddsw m4, m1 ;low: t0 high: t1 psubsw m2, m5, m0 ;low: t6 high: t7 paddsw m5, m0 ;low: t2 high: t3 shufps m1, m3, m2, q1032 punpckhwd m2, m1 punpcklwd m3, m1 ITX_MUL2X_PACK 3, 0, 6, 1567, 3784, 1 ;low: t5a high: t4a ITX_MUL2X_PACK 2, 0, 6, 3784, 1567 ;low: t7a high: t6a psubsw m1, m4, m5 ;low: t2 high: t3 paddsw m4, m5 ;low: out0 high: -out7 psubsw m5, m3, m2 ;low: t7 high: t6 paddsw m3, m2 ;low: out6 high: -out1 shufps m0, m4, m3, q3210 ;low: out0 high: -out1 shufps m3, m4, q3210 ;low: out6 high: -out7 shufps m4, m1, m5, q1032 ;low: t3 high: t7 shufps m1, m5, q3210 ;low: t2 high: t6 mova m5, [o(pw_2896x8)] psubw m2, m1, m4 ;low: t2-t3 high: t6-t7 paddw m1, m4 ;low: t2+t3 high: t6+t7 pmulhrsw m2, m5 ;low: out4 high: -out5 shufps m1, m1, q1032 pmulhrsw m1, m5 ;low: out2 high: -out3 %endmacro %macro WRITE_4X8 4 ;row[1-4] WRITE_4X4 0, 1, 4, 5, 6, %1, %2, %3, %4 lea dstq, [dstq+strideq*4] WRITE_4X4 2, 3, 4, 5, 6, %1, %2, %3, %4 %endmacro %macro INV_4X8 0 punpckhwd m4, m2, m3 punpcklwd m2, m3 punpckhwd m3, m0, m1 punpcklwd m0, m1 punpckhdq m1, m0, m2 ;low: in2 high: in3 punpckldq m0, m2 ;low: in0 high: in1 punpckldq m2, m3, m4 ;low: in4 high: in5 punpckhdq m3, m4 ;low: in6 high: in7 %endmacro %macro INV_TXFM_4X8_FN 2-3 -1 ; type1, type2, fast_thresh INV_TXFM_FN %1, %2, %3, 4x8, 8 %if %3 >= 0 %ifidn %1_%2, dct_identity mova m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] pmulhrsw m0, m1 pmulhrsw m0, [o(pw_4096)] punpckhwd m2, m0, m0 punpcklwd m0, m0 punpckhdq m1, m0, m0 punpckldq m0, m0 punpckhdq m3, m2, m2 punpckldq m2, m2 TAIL_CALL m(iadst_4x8_internal).end3 %elifidn %1_%2, identity_dct movd m0, [coeffq+16*0] punpcklwd m0, [coeffq+16*1] movd m1, [coeffq+16*2] punpcklwd m1, [coeffq+16*3] mova m2, [o(pw_2896x8)] punpckldq m0, m1 pmulhrsw m0, m2 paddw m0, m0 pmulhrsw m0, [o(pw_5793x4)] pmulhrsw m0, m2 pmulhrsw m0, [o(pw_2048)] punpcklqdq m0, m0 mova m1, m0 mova m2, m0 mova m3, m0 TAIL_CALL m(iadst_4x8_internal).end3 %elifidn %1_%2, dct_dct pshuflw m0, [coeffq], q0000 punpcklqdq m0, m0 mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 mov [coeffq], eobd pmulhrsw m0, m1 pmulhrsw m0, m1 pmulhrsw m0, [o(pw_2048)] mova m1, m0 mova m2, m0 mova m3, m0 TAIL_CALL m(iadst_4x8_internal).end4 %else ; adst_dct / flipadst_dct pshuflw m0, [coeffq], q0000 punpcklqdq m0, m0 mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 %ifidn %1, adst pmulhrsw m0, [o(iadst4_dconly1a)] %else ; flipadst pmulhrsw m0, [o(iadst4_dconly1b)] %endif mov [coeffq], eobd pmulhrsw m0, m1 pmulhrsw m0, [o(pw_2048)] mova m1, m0 mova m2, m0 mova m3, m0 TAIL_CALL m(iadst_4x8_internal).end4 %endif %endif %endmacro INV_TXFM_4X8_FN dct, dct, 0 INV_TXFM_4X8_FN dct, identity, 7 INV_TXFM_4X8_FN dct, adst INV_TXFM_4X8_FN dct, flipadst cglobal idct_4x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] call m(idct_8x4_internal).main call m(iadst_4x8_internal).inversion jmp tx2q .pass2: call .main shufps m1, m1, q1032 shufps m3, m3, q1032 mova m4, [o(pw_2048)] jmp m(iadst_4x8_internal).end2 ALIGN function_align .main: IDCT8_1D_PACKED ret INV_TXFM_4X8_FN adst, dct, 0 INV_TXFM_4X8_FN adst, adst INV_TXFM_4X8_FN adst, flipadst INV_TXFM_4X8_FN adst, identity cglobal iadst_4x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] call m(iadst_8x4_internal).main call .inversion jmp tx2q .pass2: shufps m0, m0, q1032 shufps m1, m1, q1032 call .main mova m4, [o(pw_2048)] pxor m5, m5 psubw m5, m4 .end: punpcklqdq m4, m5 .end2: pmulhrsw m0, m4 pmulhrsw m1, m4 pmulhrsw m2, m4 pmulhrsw m3, m4 .end3: pxor m5, m5 mova [coeffq+16*0], m5 mova [coeffq+16*1], m5 mova [coeffq+16*2], m5 mova [coeffq+16*3], m5 .end4: WRITE_4X8 0, 1, 2, 3 RET ALIGN function_align .main: IADST8_1D_PACKED ret ALIGN function_align .inversion: INV_4X8 ret INV_TXFM_4X8_FN flipadst, dct, 0 INV_TXFM_4X8_FN flipadst, adst INV_TXFM_4X8_FN flipadst, flipadst INV_TXFM_4X8_FN flipadst, identity cglobal iflipadst_4x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] call m(iadst_8x4_internal).main punpcklwd m4, m3, m2 punpckhwd m3, m2 punpcklwd m5, m1, m0 punpckhwd m1, m0 punpckldq m2, m3, m1 ;low: in4 high: in5 punpckhdq m3, m1 ;low: in6 high: in7 punpckldq m0, m4, m5 ;low: in0 high: in1 punpckhdq m1, m4, m5 ;low: in2 high: in3 jmp tx2q .pass2: shufps m0, m0, q1032 shufps m1, m1, q1032 call m(iadst_4x8_internal).main mova m4, m0 mova m5, m1 pshufd m0, m3, q1032 pshufd m1, m2, q1032 pshufd m2, m5, q1032 pshufd m3, m4, q1032 mova m5, [o(pw_2048)] pxor m4, m4 psubw m4, m5 jmp m(iadst_4x8_internal).end INV_TXFM_4X8_FN identity, dct, 3 INV_TXFM_4X8_FN identity, adst INV_TXFM_4X8_FN identity, flipadst INV_TXFM_4X8_FN identity, identity cglobal iidentity_4x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] mova m5, [o(pw_5793x4)] paddw m0, m0 paddw m1, m1 paddw m2, m2 paddw m3, m3 pmulhrsw m0, m5 pmulhrsw m1, m5 pmulhrsw m2, m5 pmulhrsw m3, m5 call m(iadst_4x8_internal).inversion jmp tx2q .pass2: mova m4, [o(pw_4096)] jmp m(iadst_4x8_internal).end2 %macro WRITE_8X2 5 ;coefs[1-2], tmp[1-3] movq m%3, [dstq ] movq m%4, [dstq+strideq] pxor m%5, m%5 punpcklbw m%3, m%5 ;extend byte to word punpcklbw m%4, m%5 ;extend byte to word %ifnum %1 paddw m%3, m%1 %else paddw m%3, %1 %endif %ifnum %2 paddw m%4, m%2 %else paddw m%4, %2 %endif packuswb m%3, m%4 movq [dstq ], m%3 punpckhqdq m%3, m%3 movq [dstq+strideq], m%3 %endmacro %macro WRITE_8X4 7 ;coefs[1-4], tmp[1-3] WRITE_8X2 %1, %2, %5, %6, %7 lea dstq, [dstq+strideq*2] WRITE_8X2 %3, %4, %5, %6, %7 %endmacro %macro INV_TXFM_8X4_FN 2-3 -1 ; type1, type2, fast_thresh INV_TXFM_FN %1, %2, %3, 8x4, 8 %if %3 >= 0 %ifidn %1_%2, dct_identity mova m0, [o(pw_2896x8)] pmulhrsw m1, m0, [coeffq] pmulhrsw m1, m0 paddw m1, m1 pmulhrsw m1, [o(pw_5793x4)] pmulhrsw m1, [o(pw_2048)] punpcklwd m1, m1 punpckhdq m2, m1, m1 punpckldq m1, m1 punpckhdq m3, m2, m2 punpckldq m2, m2 punpckldq m0, m1, m1 punpckhdq m1, m1 %elifidn %1_%2, identity_dct mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] mova m2, [coeffq+16*2] mova m3, [coeffq+16*3] punpckhwd m4, m0, m1 punpcklwd m0, m1 punpckhwd m5, m2, m3 punpcklwd m2, m3 punpcklwd m0, m4 punpcklwd m2, m5 punpcklqdq m0, m2 mova m4, [o(pw_2896x8)] pmulhrsw m0, m4 paddw m0, m0 pmulhrsw m0, m4 pmulhrsw m0, [o(pw_2048)] mova m1, m0 mova m2, m0 mova m3, m0 %else pshuflw m0, [coeffq], q0000 punpcklqdq m0, m0 mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 pmulhrsw m0, m1 %ifidn %2, dct mova m2, [o(pw_2048)] pmulhrsw m0, m1 pmulhrsw m0, m2 mova m1, m0 mova m2, m0 mova m3, m0 %else ; adst / flipadst pmulhrsw m2, m0, [o(iadst4_dconly2b)] pmulhrsw m0, [o(iadst4_dconly2a)] mova m1, [o(pw_2048)] pmulhrsw m0, m1 pmulhrsw m2, m1 %ifidn %2, adst punpckhqdq m1, m0, m0 punpcklqdq m0, m0 punpckhqdq m3, m2, m2 punpcklqdq m2, m2 %else ; flipadst mova m3, m0 punpckhqdq m0, m2, m2 punpcklqdq m1, m2, m2 punpckhqdq m2, m3, m3 punpcklqdq m3, m3 %endif %endif %endif TAIL_CALL m(iadst_8x4_internal).end2 %endif %endmacro INV_TXFM_8X4_FN dct, dct, 0 INV_TXFM_8X4_FN dct, adst, 0 INV_TXFM_8X4_FN dct, flipadst, 0 INV_TXFM_8X4_FN dct, identity, 3 cglobal idct_8x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] call m(idct_4x8_internal).main mova m4, [o(deint_shuf1)] mova m5, [o(deint_shuf2)] pshufb m0, m4 pshufb m1, m5 pshufb m2, m4 pshufb m3, m5 punpckhdq m4, m0, m1 punpckldq m0, m1 punpckhdq m5, m2, m3 punpckldq m2, m3 punpckhqdq m1, m0, m2 ;in1 punpcklqdq m0, m2 ;in0 punpckhqdq m3, m4, m5 ;in3 punpcklqdq m2 ,m4, m5 ;in2 jmp tx2q .pass2: call .main jmp m(iadst_8x4_internal).end ALIGN function_align .main: mova m6, [o(pd_2048)] IDCT4_1D 0, 1, 2, 3, 4, 5, 6 ret INV_TXFM_8X4_FN adst, dct INV_TXFM_8X4_FN adst, adst INV_TXFM_8X4_FN adst, flipadst INV_TXFM_8X4_FN adst, identity cglobal iadst_8x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] shufps m0, m0, q1032 shufps m1, m1, q1032 call m(iadst_4x8_internal).main punpckhwd m4, m0, m1 punpcklwd m0, m1 punpckhwd m1, m2, m3 punpcklwd m2, m3 pxor m5, m5 psubw m3, m5, m1 psubw m5, m4 punpckhdq m4, m5, m3 punpckldq m5, m3 punpckhdq m3, m0, m2 punpckldq m0, m2 punpckhwd m1, m0, m5 ;in1 punpcklwd m0, m5 ;in0 punpcklwd m2, m3, m4 ;in2 punpckhwd m3, m4 ;in3 jmp tx2q .pass2: call .main .end: mova m4, [o(pw_2048)] pmulhrsw m0, m4 pmulhrsw m1, m4 pmulhrsw m2, m4 pmulhrsw m3, m4 .end2: pxor m6, m6 mova [coeffq+16*0], m6 mova [coeffq+16*1], m6 mova [coeffq+16*2], m6 mova [coeffq+16*3], m6 .end3: WRITE_8X4 0, 1, 2, 3, 4, 5, 6 RET ALIGN function_align .main: IADST4_1D ret INV_TXFM_8X4_FN flipadst, dct INV_TXFM_8X4_FN flipadst, adst INV_TXFM_8X4_FN flipadst, flipadst INV_TXFM_8X4_FN flipadst, identity cglobal iflipadst_8x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] shufps m0, m0, q1032 shufps m1, m1, q1032 call m(iadst_4x8_internal).main punpckhwd m5, m3, m2 punpcklwd m3, m2 punpckhwd m2, m1, m0 punpcklwd m1, m0 pxor m0, m0 psubw m4, m0, m2 psubw m0, m5 punpckhdq m2, m0, m4 punpckldq m0, m4 punpckhdq m4, m3, m1 punpckldq m3, m1 punpckhwd m1, m0, m3 ;in1 punpcklwd m0, m3 ;in0 punpckhwd m3, m2, m4 ;in3 punpcklwd m2, m4 ;in2 jmp tx2q .pass2: call m(iadst_8x4_internal).main mova m4, m0 mova m5, m1 mova m0, m3 mova m1, m2 mova m2, m5 mova m3, m4 jmp m(iadst_8x4_internal).end INV_TXFM_8X4_FN identity, dct, 7 INV_TXFM_8X4_FN identity, adst INV_TXFM_8X4_FN identity, flipadst INV_TXFM_8X4_FN identity, identity cglobal iidentity_8x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] paddw m0, m0 paddw m1, m1 paddw m2, m2 paddw m3, m3 punpckhwd m4, m0, m1 punpcklwd m0, m1 punpckhwd m1, m2, m3 punpcklwd m2, m3 punpckhdq m5, m4, m1 punpckldq m4, m1 punpckhdq m3, m0, m2 punpckldq m0, m2 punpckhwd m1, m0, m4 ;in1 punpcklwd m0, m4 ;in0 punpcklwd m2, m3, m5 ;in2 punpckhwd m3, m5 ;in3 jmp tx2q .pass2: mova m4, [o(pw_5793x4)] paddw m0, m0 paddw m1, m1 paddw m2, m2 paddw m3, m3 pmulhrsw m0, m4 pmulhrsw m1, m4 pmulhrsw m2, m4 pmulhrsw m3, m4 jmp m(iadst_8x4_internal).end %macro INV_TXFM_8X8_FN 2-3 -1 ; type1, type2, fast_thresh INV_TXFM_FN %1, %2, %3, 8x8, 8 %ifidn %1_%2, dct_identity mova m0, [o(pw_2896x8)] pmulhrsw m0, [coeffq] mova m1, [o(pw_16384)] pmulhrsw m0, m1 psrlw m1, 2 pmulhrsw m0, m1 punpckhwd m7, m0, m0 punpcklwd m0, m0 pshufd m3, m0, q3333 pshufd m2, m0, q2222 pshufd m1, m0, q1111 pshufd m0, m0, q0000 call m(iadst_8x4_internal).end2 pshufd m3, m7, q3333 pshufd m2, m7, q2222 pshufd m1, m7, q1111 pshufd m0, m7, q0000 lea dstq, [dstq+strideq*2] TAIL_CALL m(iadst_8x4_internal).end3 %elif %3 >= 0 %ifidn %1, dct pshuflw m0, [coeffq], q0000 punpcklwd m0, m0 mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 mova m2, [o(pw_16384)] mov [coeffq], eobd pmulhrsw m0, m2 psrlw m2, 3 pmulhrsw m0, m1 pmulhrsw m0, m2 .end: mov r2d, 2 .end2: lea r3, [strideq*3] .loop: WRITE_8X4 0, 0, 0, 0, 1, 2, 3 lea dstq, [dstq+strideq*2] dec r2d jg .loop RET %else ; identity mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] mova m2, [coeffq+16*2] mova m3, [coeffq+16*3] punpcklwd m0, [coeffq+16*4] punpcklwd m1, [coeffq+16*5] punpcklwd m2, [coeffq+16*6] punpcklwd m3, [coeffq+16*7] punpcklwd m0, m2 punpcklwd m1, m3 punpcklwd m0, m1 pmulhrsw m0, [o(pw_2896x8)] pmulhrsw m0, [o(pw_2048)] pxor m4, m4 REPX {mova [coeffq+16*x], m4}, 0, 1, 2, 3, 4, 5, 6, 7 jmp m(inv_txfm_add_dct_dct_8x8).end %endif %endif %endmacro %macro ITX_8X8_LOAD_COEFS 0 mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] mova m2, [coeffq+16*2] mova m3, [coeffq+16*3] mova m4, [coeffq+16*4] mova m5, [coeffq+16*5] mova m6, [coeffq+16*6] %endmacro %macro IDCT8_1D_ODDHALF 7 ; src[1-4], tmp[1-2], pd_2048 ITX_MULSUB_2W %1, %4, %5, %6, %7, 799, 4017 ;t4a, t7a ITX_MULSUB_2W %3, %2, %5, %6, %7, 3406, 2276 ;t5a, t6a psubsw m%5, m%1, m%3 ;t5a paddsw m%1, m%3 ;t4 psubsw m%6, m%4, m%2 ;t6a paddsw m%4, m%2 ;t7 mova m%3, [o(pw_2896x8)] psubw m%2, m%6, m%5 ;t6a - t5a paddw m%6, m%5 ;t6a + t5a pmulhrsw m%2, m%3 ;t5 pmulhrsw m%3, m%6 ;t6 %endmacro INV_TXFM_8X8_FN dct, dct, 0 INV_TXFM_8X8_FN dct, identity, 7 INV_TXFM_8X8_FN dct, adst INV_TXFM_8X8_FN dct, flipadst cglobal idct_8x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 ITX_8X8_LOAD_COEFS call .main .pass1_end: mova m7, [o(pw_16384)] REPX {pmulhrsw x, m7}, m0, m2, m4, m6 mova [coeffq+16*6], m6 .pass1_end2: REPX {pmulhrsw x, m7}, m1, m3, m5 pmulhrsw m7, [coeffq+16*7] .pass1_end3: punpcklwd m6, m1, m5 ;10 50 11 51 12 52 13 53 punpckhwd m1, m5 ;14 54 15 55 16 56 17 57 punpckhwd m5, m0, m4 ;04 44 05 45 06 46 07 47 punpcklwd m0, m4 ;00 40 01 41 02 42 03 43 punpckhwd m4, m3, m7 ;34 74 35 75 36 76 37 77 punpcklwd m3, m7 ;30 70 31 71 32 72 33 73 punpckhwd m7, m1, m4 ;16 36 56 76 17 37 57 77 punpcklwd m1, m4 ;14 34 54 74 15 35 55 75 punpckhwd m4, m6, m3 ;12 32 52 72 13 33 53 73 punpcklwd m6, m3 ;10 30 50 70 11 31 51 71 mova [coeffq+16*5], m6 mova m6, [coeffq+16*6] punpckhwd m3, m2, m6 ;24 64 25 65 26 66 27 67 punpcklwd m2, m6 ;20 60 21 61 22 62 23 63 punpckhwd m6, m5, m3 ;06 26 46 66 07 27 47 67 punpcklwd m5, m3 ;04 24 44 64 05 25 45 65 punpckhwd m3, m0, m2 ;02 22 42 62 03 23 43 63 punpcklwd m0, m2 ;00 20 40 60 01 21 41 61 punpckhwd m2, m6, m7 ;07 17 27 37 47 57 67 77 punpcklwd m6, m7 ;06 16 26 36 46 56 66 76 mova [coeffq+16*7], m2 punpcklwd m2, m3, m4 ;02 12 22 32 42 52 62 72 punpckhwd m3, m4 ;03 13 23 33 43 53 63 73 punpcklwd m4, m5, m1 ;04 14 24 34 44 54 64 74 punpckhwd m5, m1 ;05 15 25 35 45 55 65 75 mova m7, [coeffq+16*5] punpckhwd m1, m0, m7 ;01 11 21 31 41 51 61 71 punpcklwd m0, m7 ;00 10 20 30 40 50 60 70 jmp tx2q .pass2: call .main .end: mova m7, [o(pw_2048)] REPX {pmulhrsw x, m7}, m0, m2, m4, m6 mova [coeffq+16*6], m6 .end2: REPX {pmulhrsw x, m7}, m1, m3, m5 pmulhrsw m7, [coeffq+16*7] mova [coeffq+16*5], m5 mova [coeffq+16*7], m7 .end3: WRITE_8X4 0, 1, 2, 3, 5, 6, 7 lea dstq, [dstq+strideq*2] WRITE_8X4 4, [coeffq+16*5], [coeffq+16*6], [coeffq+16*7], 5, 6, 7 pxor m7, m7 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 ret ALIGN function_align .main: mova [coeffq+16*6], m3 mova [coeffq+16*5], m1 mova m7, [o(pd_2048)] IDCT4_1D 0, 2, 4, 6, 1, 3, 7 mova m3, [coeffq+16*5] mova [coeffq+16*5], m2 mova m2, [coeffq+16*6] mova [coeffq+16*6], m4 mova m4, [coeffq+16*7] mova [coeffq+16*7], m6 IDCT8_1D_ODDHALF 3, 2, 5, 4, 1, 6, 7 mova m6, [coeffq+16*7] psubsw m7, m0, m4 ;out7 paddsw m0, m4 ;out0 mova [coeffq+16*7], m7 mova m1, [coeffq+16*5] psubsw m4, m6, m3 ;out4 paddsw m3, m6 ;out3 mova m7, [coeffq+16*6] psubsw m6, m1, m5 ;out6 paddsw m1, m5 ;out1 psubsw m5, m7, m2 ;out5 paddsw m2, m7 ;out2 ret INV_TXFM_8X8_FN adst, dct INV_TXFM_8X8_FN adst, adst INV_TXFM_8X8_FN adst, flipadst INV_TXFM_8X8_FN adst, identity cglobal iadst_8x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 ITX_8X8_LOAD_COEFS call .main mova m7, [o(pw_16384)] REPX {pmulhrsw x, m7}, m0, m2, m4, m6 mova [coeffq+16*6], m6 pxor m6, m6 psubw m6, m7 mova m7, m6 jmp m(idct_8x8_internal).pass1_end2 ALIGN function_align .pass2: call .main mova m7, [o(pw_2048)] REPX {pmulhrsw x, m7}, m0, m2, m4, m6 mova [coeffq+16*6], m6 pxor m6, m6 psubw m6, m7 mova m7, m6 jmp m(idct_8x8_internal).end2 ALIGN function_align .main: mova [coeffq+16*6], m3 mova [coeffq+16*5], m4 mova m7, [o(pd_2048)] ITX_MULSUB_2W 5, 2, 3, 4, 7, 1931, 3612 ;t3a, t2a ITX_MULSUB_2W 1, 6, 3, 4, 7, 3920, 1189 ;t7a, t6a paddsw m3, m2, m6 ;t2 psubsw m2, m6 ;t6 paddsw m4, m5, m1 ;t3 psubsw m5, m1 ;t7 ITX_MULSUB_2W 5, 2, 1, 6, 7, 3784, 1567 ;t6a, t7a mova m6, [coeffq+16*5] mova [coeffq+16*5], m5 mova m1, [coeffq+16*6] mova [coeffq+16*6], m2 mova m5, [coeffq+16*7] mova [coeffq+16*7], m3 ITX_MULSUB_2W 5, 0, 2, 3, 7, 401, 4076 ;t1a, t0a ITX_MULSUB_2W 1, 6, 2, 3, 7, 3166, 2598 ;t5a, t4a psubsw m2, m0, m6 ;t4 paddsw m0, m6 ;t0 paddsw m3, m5, m1 ;t1 psubsw m5, m1 ;t5 ITX_MULSUB_2W 2, 5, 1, 6, 7, 1567, 3784 ;t5a, t4a mova m7, [coeffq+16*7] paddsw m1, m3, m4 ;-out7 psubsw m3, m4 ;t3 mova [coeffq+16*7], m1 psubsw m4, m0, m7 ;t2 paddsw m0, m7 ;out0 mova m6, [coeffq+16*5] mova m7, [coeffq+16*6] paddsw m1, m5, m6 ;-out1 psubsw m5, m6 ;t6 paddsw m6, m2, m7 ;out6 psubsw m2, m7 ;t7 paddw m7, m4, m3 ;t2 + t3 psubw m4, m3 ;t2 - t3 paddw m3, m5, m2 ;t6 + t7 psubw m5, m2 ;t6 - t7 mova m2, [o(pw_2896x8)] pmulhrsw m4, m2 ;out4 pmulhrsw m5, m2 ;-out5 pmulhrsw m7, m2 ;-out3 pmulhrsw m2, m3 ;out2 mova m3, m7 ret INV_TXFM_8X8_FN flipadst, dct INV_TXFM_8X8_FN flipadst, adst INV_TXFM_8X8_FN flipadst, flipadst INV_TXFM_8X8_FN flipadst, identity cglobal iflipadst_8x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 ITX_8X8_LOAD_COEFS call m(iadst_8x8_internal).main mova m7, [o(pw_m16384)] pmulhrsw m1, m7 mova [coeffq+16*6], m1 mova m1, m6 mova m6, m2 pmulhrsw m2, m5, m7 mova m5, m6 mova m6, m4 pmulhrsw m4, m3, m7 mova m3, m6 mova m6, m0 mova m0, m7 pxor m7, m7 psubw m7, m0 pmulhrsw m0, [coeffq+16*7] REPX {pmulhrsw x, m7}, m1, m3, m5 pmulhrsw m7, m6 jmp m(idct_8x8_internal).pass1_end3 ALIGN function_align .pass2: call m(iadst_8x8_internal).main mova m7, [o(pw_2048)] REPX {pmulhrsw x, m7}, m0, m2, m4, m6 mova [coeffq+16*5], m2 mova m2, m0 pxor m0, m0 psubw m0, m7 mova m7, m2 pmulhrsw m1, m0 pmulhrsw m2, m5, m0 mova [coeffq+16*6], m1 mova m5, m4 mova m1, m6 pmulhrsw m4, m3, m0 pmulhrsw m0, [coeffq+16*7] mova m3, m5 mova [coeffq+16*7], m7 jmp m(idct_8x8_internal).end3 INV_TXFM_8X8_FN identity, dct, 7 INV_TXFM_8X8_FN identity, adst INV_TXFM_8X8_FN identity, flipadst INV_TXFM_8X8_FN identity, identity cglobal iidentity_8x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] mova m2, [coeffq+16*2] mova m3, [coeffq+16*3] mova m4, [coeffq+16*4] mova m5, [coeffq+16*5] mova m7, [coeffq+16*7] jmp m(idct_8x8_internal).pass1_end3 ALIGN function_align .pass2: mova m7, [o(pw_4096)] REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 pmulhrsw m7, [coeffq+16*7] mova [coeffq+16*5], m5 mova [coeffq+16*6], m6 mova [coeffq+16*7], m7 jmp m(idct_8x8_internal).end3