ref: 8c71f0f0d3789b796540fec33641125f187537dd
dir: /src/x86/cdef_avx512.asm/
; Copyright © 2020, VideoLAN and dav1d authors ; Copyright © 2020, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: ; ; 1. Redistributions of source code must retain the above copyright notice, this ; list of conditions and the following disclaimer. ; ; 2. Redistributions in binary form must reproduce the above copyright notice, ; this list of conditions and the following disclaimer in the documentation ; and/or other materials provided with the distribution. ; ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %include "config.asm" %include "ext/x86/x86inc.asm" %if HAVE_AVX512ICL && ARCH_X86_64 %macro DUP4 1-* %rep %0 times 4 db %1 %rotate 1 %endrep %endmacro %macro DIRS 16 ; cdef_directions[] %rep 4 + 16 + 4 ; 6 7 0 1 2 3 4 5 6 7 0 1 ; masking away unused bits allows us to use a single vpaddd {1to16} ; instruction instead of having to do vpbroadcastd + paddb db %13 & 0x3f, -%13 & 0x3f %rotate 1 %endrep %endmacro SECTION_RODATA 64 lut_perm_4x4: db 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 db 16, 17, 0, 1, 2, 3, 4, 5, 18, 19, 8, 9, 10, 11, 12, 13 db 20, 21, 80, 81, 82, 83, 84, 85, 22, 23, 32, 33, 34, 35, 36, 37 db 98, 99,100,101,102,103,104,105, 50, 51, 52, 53, 54, 55, 56, 57 lut_perm_4x8a: db 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 db 96, 97, 0, 1, 2, 3, 4, 5, 98, 99, 8, 9, 10, 11, 12, 13 lut_perm_4x8b:db 100,101, 16, 17, 18, 19, 20, 21,102,103, 24, 25, 26, 27, 28, 29 db 104,105, 32, 33, 34, 35, 36, 37,106,107, 40, 41, 42, 43, 44, 45 db 108,109, 48, 49, 50, 51, 52, 53,110,111, 56, 57, 58, 59, 60, 61 db 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95 pd_01234567: dd 0, 1, 2, 3, 4, 5, 6, 7 lut_perm_8x8a: db 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 db -1, -1, 34, 35, 36, 37, 38, 39, -1, -1, 50, 51, 52, 53, 54, 55 db -1, -1, 66, 67, 68, 69, 70, 71, -1, -1, 82, 83, 84, 85, 86, 87 db 96, 97, 98, 99,100,101,102,103,112,113,114,115,116,117,118,119 lut_perm_8x8b: db 4, 5, 6, 7, 8, 9, 10, 11, 20, 21, 22, 23, 24, 25, 26, 27 db 36, 37, 38, 39, 40, 41, 42, 43, 52, 53, 54, 55, 56, 57, 58, 59 db 68, 69, 70, 71, 72, 73, 74, 75, 84, 85, 86, 87, 88, 89, 90, 91 db 100,101,102,103,104,105,106,107,116,117,118,119,120,121,122,123 edge_mask: dq 0x00003c3c3c3c0000, 0x00003f3f3f3f0000 ; 0000, 0001 dq 0x0000fcfcfcfc0000, 0x0000ffffffff0000 ; 0010, 0011 dq 0x00003c3c3c3c3c3c, 0x00003f3f3f3f3f3f ; 0100, 0101 dq 0x0000fcfcfcfcfcfc, 0x0000ffffffffffff ; 0110, 0111 dq 0x3c3c3c3c3c3c0000, 0x3f3f3f3f3f3f0000 ; 1000, 1001 dq 0xfcfcfcfcfcfc0000, 0xffffffffffff0000 ; 1010, 1011 dq 0x3c3c3c3c3c3c3c3c, 0x3f3f3f3f3f3f3f3f ; 1100, 1101 dq 0xfcfcfcfcfcfcfcfc, 0xffffffffffffffff ; 1110, 1111 px_idx: DUP4 18, 19, 20, 21, 26, 27, 28, 29, 34, 35, 36, 37, 42, 43, 44, 45 cdef_dirs: DIRS -7,-14, 1, -6, 1, 2, 1, 10, 9, 18, 8, 17, 8, 16, 8, 15 gf_shr: dq 0x0102040810204080, 0x0102040810204080 ; >> 0, >> 0 dq 0x0204081020408000, 0x0408102040800000 ; >> 1, >> 2 dq 0x0810204080000000, 0x1020408000000000 ; >> 3, >> 4 dq 0x2040800000000000, 0x4080000000000000 ; >> 5, >> 6 times 16 db 0 ; realign (introduced by cdef_dirs) end_perm_w8clip:db 0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30 db 32, 36, 40, 44, 34, 38, 42, 46, 48, 52, 56, 60, 50, 54, 58, 62 db 1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31 db 33, 37, 41, 45, 35, 39, 43, 47, 49, 53, 57, 61, 51, 55, 59, 63 end_perm: db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 db 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 pri_tap: db 64, 64, 32, 32, 48, 48, 48, 48 ; left-shifted by 4 sec_tap: db 32, 32, 16, 16 pd_268435568: dd 268435568 SECTION .text %if WIN64 DECLARE_REG_TMP 5, 6 %else DECLARE_REG_TMP 8, 5 %endif ; lut: ; t0 t1 t2 t3 t4 t5 t6 t7 ; T0 T1 T2 T3 T4 T5 T6 T7 ; L0 L1 00 01 02 03 04 05 ; L2 L3 10 11 12 13 14 15 ; L4 L5 20 21 22 23 24 25 ; L6 L7 30 31 32 33 34 35 ; 4e 4f 40 41 42 43 44 45 ; 5e 5f 50 51 52 53 54 55 INIT_ZMM avx512icl cglobal cdef_filter_4x4, 4, 8, 13, dst, stride, left, top, pri, sec, dir, damping, edge %define base r7-edge_mask movq xmm0, [dstq+strideq*0] movhps xmm0, [dstq+strideq*1] lea r7, [edge_mask] movq xmm1, [topq+strideq*0-2] movhps xmm1, [topq+strideq*1-2] mov r6d, edgem vinserti32x4 ym0, ymm0, [leftq], 1 lea r2, [strideq*3] vinserti32x4 ym1, ymm1, [dstq+strideq*2], 1 mova m5, [base+lut_perm_4x4] vinserti32x4 m0, [dstq+r2], 2 test r6b, 0x08 ; avoid buffer overread jz .main lea r3, [dstq+strideq*4-4] vinserti32x4 m1, [r3+strideq*0], 2 vinserti32x4 m0, [r3+strideq*1], 3 .main: movifnidn prid, prim mov t0d, dirm mova m3, [base+px_idx] mov r3d, dampingm vpermi2b m5, m0, m1 ; lut vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) pxor m7, m7 lea r3, [r7+r3*8] ; gf_shr + (damping - 30) * 8 vpermb m6, m3, m5 ; px cmp r6d, 0x0f jne .mask_edges ; mask edges only if required test prid, prid jz .sec_only vpaddd m1, m3, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir vpermb m1, m1, m5 ; k0p0 k0p1 k1p0 k1p1 %macro CDEF_FILTER_4x4_PRI 0 vpcmpub k1, m6, m1, 6 ; px > pN psubb m2, m1, m6 lzcnt r6d, prid vpsubb m2{k1}, m6, m1 ; abs(diff) vpbroadcastb m4, prid and prid, 1 vgf2p8affineqb m9, m2, [r3+r6*8] {1to8}, 0 ; abs(diff) >> shift movifnidn t1d, secm vpbroadcastd m10, [base+pri_tap+priq*4] vpsubb m10{k1}, m7, m10 ; apply_sign(pri_tap) psubusb m4, m9 ; imax(0, pri_strength - (abs(diff) >> shift))) pminub m2, m4 vpdpbusd m0, m2, m10 ; sum %endmacro CDEF_FILTER_4x4_PRI test t1d, t1d ; sec jz .end_no_clip call .sec .end_clip: pminub m4, m6, m1 pmaxub m1, m6 pminub m5, m2, m3 pmaxub m2, m3 pminub m4, m5 pmaxub m2, m1 psrldq m1, m4, 2 psrldq m3, m2, 2 pminub m1, m4 vpcmpw k1, m0, m7, 1 vpshldd m6, m0, 8 pmaxub m2, m3 pslldq m3, m1, 1 psubw m7, m0 paddusw m0, m6 ; clip >0xff vpsubusw m0{k1}, m6, m7 ; clip <0x00 pslldq m4, m2, 1 pminub m1, m3 pmaxub m2, m4 pmaxub m0, m1 pminub m0, m2 jmp .end .sec_only: movifnidn t1d, secm call .sec .end_no_clip: vpshldd m6, m0, 8 ; (px << 8) + ((sum > -8) << 4) paddw m0, m6 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) .end: mova xm1, [base+end_perm] vpermb m0, m1, m0 ; output in bits 8-15 of each dword movd [dstq+strideq*0], xm0 pextrd [dstq+strideq*1], xm0, 1 pextrd [dstq+strideq*2], xm0, 2 pextrd [dstq+r2 ], xm0, 3 RET .mask_edges_sec_only: movifnidn t1d, secm call .mask_edges_sec jmp .end_no_clip ALIGN function_align .mask_edges: vpbroadcastq m8, [base+edge_mask+r6*8] test prid, prid jz .mask_edges_sec_only vpaddd m2, m3, [base+cdef_dirs+(t0+2)*4] {1to16} vpshufbitqmb k1, m8, m2 ; index in-range mova m1, m6 vpermb m1{k1}, m2, m5 CDEF_FILTER_4x4_PRI test t1d, t1d jz .end_no_clip call .mask_edges_sec jmp .end_clip .mask_edges_sec: vpaddd m4, m3, [base+cdef_dirs+(t0+4)*4] {1to16} vpaddd m9, m3, [base+cdef_dirs+(t0+0)*4] {1to16} vpshufbitqmb k1, m8, m4 mova m2, m6 vpermb m2{k1}, m4, m5 vpshufbitqmb k1, m8, m9 mova m3, m6 vpermb m3{k1}, m9, m5 jmp .sec_main ALIGN function_align .sec: vpaddd m2, m3, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 vpaddd m3, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 vpermb m2, m2, m5 ; k0s0 k0s1 k1s0 k1s1 vpermb m3, m3, m5 ; k0s2 k0s3 k1s2 k1s3 .sec_main: vpbroadcastd m8, [base+sec_tap] vpcmpub k1, m6, m2, 6 psubb m4, m2, m6 vpbroadcastb m12, t1d lzcnt t1d, t1d vpsubb m4{k1}, m6, m2 vpcmpub k2, m6, m3, 6 vpbroadcastq m11, [r3+t1*8] gf2p8affineqb m10, m4, m11, 0 psubb m5, m3, m6 mova m9, m8 vpsubb m8{k1}, m7, m8 psubusb m10, m12, m10 vpsubb m5{k2}, m6, m3 pminub m4, m10 vpdpbusd m0, m4, m8 gf2p8affineqb m11, m5, m11, 0 vpsubb m9{k2}, m7, m9 psubusb m12, m11 pminub m5, m12 vpdpbusd m0, m5, m9 ret DECLARE_REG_TMP 2, 7 ; lut top lut bottom ; t0 t1 t2 t3 t4 t5 t6 t7 L4 L5 20 21 22 23 24 25 ; T0 T1 T2 T3 T4 T5 T6 T7 L6 L7 30 31 32 33 34 35 ; L0 L1 00 01 02 03 04 05 L8 L9 40 41 42 43 44 45 ; L2 L3 10 11 12 13 14 15 La Lb 50 51 52 53 54 55 ; L4 L5 20 21 22 23 24 25 Lc Ld 60 61 62 63 64 65 ; L6 L7 30 31 32 33 34 35 Le Lf 70 71 72 73 74 75 ; L8 L9 40 41 42 43 44 45 8e 8f 80 81 82 83 84 85 ; La Lb 50 51 52 53 54 55 9e 9f 90 91 92 93 94 95 cglobal cdef_filter_4x8, 4, 9, 22, dst, stride, left, top, \ pri, sec, dir, damping, edge %define base r8-edge_mask vpbroadcastd ym21, strided mov r6d, edgem lea r8, [edge_mask] movq xm1, [topq+strideq*0-2] pmulld ym21, [base+pd_01234567] kxnorb k1, k1, k1 movq xm2, [topq+strideq*1-2] vpgatherdq m0{k1}, [dstq+ym21] ; +0+1 +2+3 +4+5 +6+7 mova m14, [base+lut_perm_4x8a] movu m15, [base+lut_perm_4x8b] test r6b, 0x08 ; avoid buffer overread jz .main lea r7, [dstq+strideq*8-2] vinserti32x4 ym1, [r7+strideq*0], 1 vinserti32x4 ym2, [r7+strideq*1], 1 .main: punpcklqdq ym1, ym2 vinserti32x4 m1, [leftq], 2 ; -2-1 +8+9 left ____ movifnidn prid, prim mov t0d, dirm mova m16, [base+px_idx] mov r3d, dampingm vpermi2b m14, m0, m1 ; lut top vpermi2b m15, m0, m1 ; lut bottom vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) pxor m20, m20 lea r3, [r8+r3*8] ; gf_shr + (damping - 30) * 8 vpermb m2, m16, m14 ; pxt vpermb m3, m16, m15 ; pxb mova m1, m0 cmp r6b, 0x0f jne .mask_edges ; mask edges only if required test prid, prid jz .sec_only vpaddd m6, m16, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir vpermb m4, m6, m14 ; pNt k0p0 k0p1 k1p0 k1p1 vpermb m5, m6, m15 ; pNb %macro CDEF_FILTER_4x8_PRI 0 vpcmpub k1, m2, m4, 6 ; pxt > pNt vpcmpub k2, m3, m5, 6 ; pxb > pNb psubb m6, m4, m2 psubb m7, m5, m3 lzcnt r6d, prid vpsubb m6{k1}, m2, m4 ; abs(diff_top) vpsubb m7{k2}, m3, m5 ; abs(diff_bottom) vpbroadcastb m13, prid vpbroadcastq m9, [r3+r6*8] and prid, 1 vpbroadcastd m11, [base+pri_tap+priq*4] vgf2p8affineqb m8, m6, m9, 0 ; abs(dt) >> shift vgf2p8affineqb m9, m7, m9, 0 ; abs(db) >> shift mova m10, m11 movifnidn t1d, secm vpsubb m10{k1}, m20, m11 ; apply_sign(pri_tap_top) vpsubb m11{k2}, m20, m11 ; apply_sign(pri_tap_bottom) psubusb m12, m13, m8 ; imax(0, pri_strength - (abs(dt) >> shift))) psubusb m13, m13, m9 ; imax(0, pri_strength - (abs(db) >> shift))) pminub m6, m12 pminub m7, m13 vpdpbusd m0, m6, m10 ; sum top vpdpbusd m1, m7, m11 ; sum bottom %endmacro CDEF_FILTER_4x8_PRI test t1d, t1d ; sec jz .end_no_clip call .sec .end_clip: pminub m10, m4, m2 pminub m12, m6, m8 pminub m11, m5, m3 pminub m13, m7, m9 pmaxub m4, m2 pmaxub m6, m8 pmaxub m5, m3 pmaxub m7, m9 pminub m10, m12 pminub m11, m13 pmaxub m4, m6 pmaxub m5, m7 mov r2d, 0xAAAAAAAA kmovd k1, r2d kxnorb k2, k2, k2 ; hw lw vpshrdd m12, m0, m1, 16 ; m1lw m0hw vpshrdd m6, m10, m11, 16 ; m11lw m10hw vpshrdd m8, m4, m5, 16 ; m5lw m4hw vpblendmw m7{k1}, m10, m11 ; m11hw m10lw vpblendmw m9{k1}, m4, m5 ; m5hw m4lw vpblendmw m4{k1}, m0, m12 ; m1lw m0lw vpblendmw m5{k1}, m12, m1 ; m1hw m0hw vpshrdd m2, m3, 16 pminub m6, m7 pmaxub m8, m9 mova ym14, [base+end_perm] vpcmpw k1, m4, m20, 1 vpshldw m2, m5, 8 pslldq m7, m6, 1 pslldq m9, m8, 1 psubw m5, m20, m4 paddusw m0, m4, m2 ; clip >0xff pminub m6, m7 pmaxub m8, m9 psubusw m0{k1}, m2, m5 ; clip <0x00 pmaxub m0, m6 pminub m0, m8 vpermb m0, m14, m0 vpscatterdd [dstq+ym21]{k2}, ym0 RET .sec_only: movifnidn t1d, secm call .sec .end_no_clip: mova ym4, [base+end_perm] kxnorb k1, k1, k1 vpshldd m2, m0, 8 ; (px << 8) + ((sum > -8) << 4) vpshldd m3, m1, 8 paddw m0, m2 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) paddw m1, m3 pslld m0, 16 vpshrdd m0, m1, 16 vpermb m0, m4, m0 ; output in bits 8-15 of each word vpscatterdd [dstq+ym21]{k1}, ym0 RET .mask_edges_sec_only: movifnidn t1d, secm call .mask_edges_sec jmp .end_no_clip ALIGN function_align .mask_edges: mov t1d, r6d or r6d, 8 ; top 4x4 has bottom or t1d, 4 ; bottom 4x4 has top vpbroadcastq m17, [base+edge_mask+r6*8] vpbroadcastq m18, [base+edge_mask+t1*8] test prid, prid jz .mask_edges_sec_only vpaddd m6, m16, [base+cdef_dirs+(t0+2)*4] {1to16} vpshufbitqmb k1, m17, m6 ; index in-range vpshufbitqmb k2, m18, m6 mova m4, m2 mova m5, m3 vpermb m4{k1}, m6, m14 vpermb m5{k2}, m6, m15 CDEF_FILTER_4x8_PRI test t1d, t1d jz .end_no_clip call .mask_edges_sec jmp .end_clip .mask_edges_sec: vpaddd m10, m16, [base+cdef_dirs+(t0+4)*4] {1to16} vpaddd m11, m16, [base+cdef_dirs+(t0+0)*4] {1to16} vpshufbitqmb k1, m17, m10 vpshufbitqmb k2, m18, m10 vpshufbitqmb k3, m17, m11 vpshufbitqmb k4, m18, m11 mova m6, m2 mova m7, m3 mova m8, m2 mova m9, m3 vpermb m6{k1}, m10, m14 vpermb m7{k2}, m10, m15 vpermb m8{k3}, m11, m14 vpermb m9{k4}, m11, m15 jmp .sec_main ALIGN function_align .sec: vpaddd m8, m16, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 vpaddd m9, m16, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 vpermb m6, m8, m14 ; pNt k0s0 k0s1 k1s0 k1s1 vpermb m7, m8, m15 ; pNb vpermb m8, m9, m14 ; pNt k0s2 k0s3 k1s2 k1s3 vpermb m9, m9, m15 ; pNb .sec_main: vpbroadcastb m18, t1d lzcnt t1d, t1d vpcmpub k1, m2, m6, 6 vpcmpub k2, m3, m7, 6 vpcmpub k3, m2, m8, 6 vpcmpub k4, m3, m9, 6 vpbroadcastq m17, [r3+t1*8] psubb m10, m6, m2 psubb m11, m7, m3 psubb m12, m8, m2 psubb m13, m9, m3 vpsubb m10{k1}, m2, m6 ; abs(dt0) vpsubb m11{k2}, m3, m7 ; abs(db0) vpsubb m12{k3}, m2, m8 ; abs(dt1) vpsubb m13{k4}, m3, m9 ; abs(db1) vpbroadcastd m19, [base+sec_tap] gf2p8affineqb m14, m10, m17, 0 ; abs(dt0) >> shift gf2p8affineqb m15, m11, m17, 0 ; abs(db0) >> shift gf2p8affineqb m16, m12, m17, 0 ; abs(dt1) >> shift gf2p8affineqb m17, m13, m17, 0 ; abs(db1) >> shift psubusb m14, m18, m14 ; imax(0, sec_strength - (abs(dt0) >> shift))) psubusb m15, m18, m15 ; imax(0, sec_strength - (abs(db0) >> shift))) psubusb m16, m18, m16 ; imax(0, sec_strength - (abs(dt1) >> shift))) psubusb m17, m18, m17 ; imax(0, sec_strength - (abs(db1) >> shift))) pminub m10, m14 pminub m11, m15 pminub m12, m16 pminub m13, m17 mova m14, m19 mova m15, m19 mova m16, m19 vpsubb m14{k1}, m20, m19 ; apply_sign(sec_tap_top_0) vpsubb m15{k2}, m20, m19 ; apply_sign(sec_tap_bottom_0) vpsubb m16{k3}, m20, m19 ; apply_sign(sec_tap_top_1) vpsubb m19{k4}, m20, m19 ; apply_sign(sec_tap_bottom_1) vpdpbusd m0, m10, m14 vpdpbusd m1, m11, m15 vpdpbusd m0, m12, m16 vpdpbusd m1, m13, m19 ret ; lut tl lut tr ; t0 t1 t2 t3 t4 t5 t6 t7 t6 t7 t8 t9 ta tb tc td ; T0 T1 T2 T3 T4 T5 T6 T7 T6 T7 T8 T9 TA TB TC TD ; L0 L1 00 01 02 03 04 05 04 05 06 07 08 09 0a 0b ; L2 L3 10 11 12 13 14 15 14 15 16 17 18 19 1a 1b ; L4 L5 20 21 22 23 24 25 24 25 26 27 28 29 2a 2b ; L6 L7 30 31 32 33 34 35 34 35 36 37 38 39 3a 3b ; L8 L9 40 41 42 43 44 45 44 45 46 47 48 49 4a 4b ; La Lb 50 51 52 53 54 55 54 55 56 57 58 59 5a 5b ; lut bl lut br ; L4 L5 20 21 22 23 24 25 24 25 26 27 28 29 2a 2b ; L6 L7 30 31 32 33 34 35 34 35 36 37 38 39 3a 3b ; L8 L9 40 41 42 43 44 45 44 45 46 47 48 49 4a 4b ; La Lb 50 51 52 53 54 55 54 55 56 57 58 59 5a 5b ; Lc Ld 60 61 62 63 64 65 64 65 66 67 68 69 6a 6b ; Le Lf 70 71 72 73 74 75 74 75 76 77 78 79 7a 7b ; 8e 8f 80 81 82 83 84 85 84 85 86 87 88 89 8a 8b ; 9e 9f 90 91 92 93 94 95 94 95 96 97 98 99 9a 9b cglobal cdef_filter_8x8, 4, 11, 32, 4*64, dst, stride, left, top, \ pri, sec, dir, damping, edge %define base r8-edge_mask mov r6d, edgem lea r10, [dstq+strideq*4-2] movu xmm0, [topq+strideq*0-2] movu xmm1, [dstq+strideq*2-2] movu xmm2, [r10 +strideq*2 ] lea r8, [edge_mask] lea r9, [strideq*3] pmovzxwq m10, [leftq-4] vinserti32x4 ym0, ymm0, [topq+strideq*1-2], 1 vinserti32x4 ym1, ymm1, [dstq+r9 -2], 1 vinserti32x4 ym2, ymm2, [r10 +r9 ], 1 lea r7, [r10 +strideq*4 ] pmovzxwq m11, [leftq+4] vinserti32x4 m0, [dstq+strideq*0-2], 2 vinserti32x4 m1, [r10 +strideq*0 ], 2 mova m12, [base+lut_perm_8x8a] movu m13, [base+lut_perm_8x8b] vinserti32x4 m0, [dstq+strideq*1-2], 3 vinserti32x4 m1, [r10 +strideq*1 ], 3 test r6b, 0x08 ; avoid buffer overread jz .main vinserti32x4 m2, [r7 +strideq*0], 2 vinserti32x4 m2, [r7 +strideq*1], 3 .main: mov t1d, 0x11111100 mova m14, m12 mova m15, m13 kmovd k1, t1d kshiftrd k2, k1, 8 movifnidn prid, prim mov t0d, dirm mova m30, [base+px_idx] mov r3d, dampingm vpermi2b m12, m0, m1 ; lut tl vpermi2b m14, m1, m2 ; lut bl vpermi2b m13, m0, m1 ; lut tr vpermi2b m15, m1, m2 ; lut br vpblendmw m12{k1}, m12, m10 vpblendmw m14{k2}, m14, m11 vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) pxor m31, m31 lea r3, [r8+r3*8] ; gf_shr + (damping - 30) * 8 vpermb m4, m30, m12 ; pxtl vpermb m5, m30, m13 ; pxtr vpermb m6, m30, m14 ; pxbl vpermb m7, m30, m15 ; pxbr mova m1, m0 mova m2, m0 mova m3, m0 cmp r6b, 0x0f jne .mask_edges ; mask edges only if required test prid, prid jz .sec_only vpaddd m11, m30, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir vpermb m8, m11, m12 ; pNtl k0p0 k0p1 k1p0 k1p1 vpermb m9, m11, m13 ; pNtr vpermb m10, m11, m14 ; pNbl vpermb m11, m11, m15 ; pNbr %macro CDEF_FILTER_8x8_PRI 0 vpcmpub k1, m4, m8, 6 ; pxtl > pNtl vpcmpub k2, m5, m9, 6 ; pxtr > pNtr vpcmpub k3, m6, m10, 6 ; pxbl > pNbl vpcmpub k4, m7, m11, 6 ; pxbr > pNbr psubb m16, m8, m4 psubb m17, m9, m5 psubb m18, m10, m6 psubb m19, m11, m7 lzcnt r6d, prid vpsubb m16{k1}, m4, m8 ; abs(diff_tl) vpsubb m17{k2}, m5, m9 ; abs(diff_tr) vpsubb m18{k3}, m6, m10 ; abs(diff_bl) vpsubb m19{k4}, m7, m11 ; abs(diff_br) vpbroadcastq m28, [r3+r6*8] vpbroadcastb m29, prid and prid, 1 vpbroadcastd m27, [base+pri_tap+priq*4] vgf2p8affineqb m20, m16, m28, 0 ; abs(dtl) >> shift vgf2p8affineqb m21, m17, m28, 0 ; abs(dtr) >> shift vgf2p8affineqb m22, m18, m28, 0 ; abs(dbl) >> shift vgf2p8affineqb m23, m19, m28, 0 ; abs(dbl) >> shift mova m24, m27 mova m25, m27 mova m26, m27 movifnidn t1d, secm vpsubb m24{k1}, m31, m27 ; apply_sign(pri_tap_tl) vpsubb m25{k2}, m31, m27 ; apply_sign(pri_tap_tr) vpsubb m26{k3}, m31, m27 ; apply_sign(pri_tap_tl) vpsubb m27{k4}, m31, m27 ; apply_sign(pri_tap_tr) psubusb m20, m29, m20 ; imax(0, pri_strength - (abs(dtl) >> shift))) psubusb m21, m29, m21 ; imax(0, pri_strength - (abs(dtr) >> shift))) psubusb m22, m29, m22 ; imax(0, pri_strength - (abs(dbl) >> shift))) psubusb m23, m29, m23 ; imax(0, pri_strength - (abs(dbr) >> shift))) pminub m16, m20 pminub m17, m21 pminub m18, m22 pminub m19, m23 vpdpbusd m0, m16, m24 ; sum tl vpdpbusd m1, m17, m25 ; sum tr vpdpbusd m2, m18, m26 ; sum bl vpdpbusd m3, m19, m27 ; sum br %endmacro CDEF_FILTER_8x8_PRI test t1d, t1d ; sec jz .end_no_clip call .sec .end_clip: pminub m20, m8, m4 pminub m24, m12, m16 pminub m21, m9, m5 pminub m25, m13, m17 pminub m22, m10, m6 pminub m26, m14, m18 pminub m23, m11, m7 pminub m27, m15, m19 pmaxub m8, m4 pmaxub m12, m16 pmaxub m9, m5 pmaxub m13, m17 pmaxub m10, m6 pmaxub m14, m18 pmaxub m11, m7 pmaxub m15, m19 pminub m20, m24 pminub m21, m25 pminub m22, m26 pminub m23, m27 pmaxub m8, m12 pmaxub m9, m13 pmaxub m10, m14 pmaxub m11, m15 mov r2d, 0xAAAAAAAA kmovd k1, r2d vpshrdd m24, m0, m1, 16 vpshrdd m25, m2, m3, 16 vpshrdd m12, m20, m21, 16 vpshrdd m14, m22, m23, 16 vpshrdd m16, m8, m9, 16 vpshrdd m18, m10, m11, 16 vpblendmw m13{k1}, m20, m21 vpblendmw m15{k1}, m22, m23 vpblendmw m17{k1}, m8, m9 vpblendmw m19{k1}, m10, m11 vpblendmw m20{k1}, m0, m24 vpblendmw m21{k1}, m24, m1 vpblendmw m22{k1}, m2, m25 vpblendmw m23{k1}, m25, m3 vpshrdd m4, m5, 16 vpshrdd m6, m7, 16 pminub m12, m13 pminub m14, m15 pmaxub m16, m17 pmaxub m18, m19 mova m8, [base+end_perm_w8clip] vpcmpw k2, m20, m31, 1 vpcmpw k3, m22, m31, 1 vpshldw m4, m21, 8 vpshldw m6, m23, 8 kunpckdq k1, k1, k1 kxnorb k4, k4, k4 vpshrdw m11, m12, m14, 8 vpshrdw m15, m16, m18, 8 vpblendmb m13{k1}, m12, m14 vpblendmb m17{k1}, m16, m18 psubw m21, m31, m20 psubw m23, m31, m22 paddusw m0, m20, m4 ; clip >0xff paddusw m1, m22, m6 pminub m11, m13 pmaxub m15, m17 psubusw m0{k2}, m4, m21 ; clip <0x00 psubusw m1{k3}, m6, m23 psrlw m0, 8 vmovdqu8 m0{k1}, m1 pmaxub m0, m11 pminub m0, m15 vpermb m0, m8, m0 add r10, 2 vextracti32x4 xm1, m0, 1 vextracti32x4 xm2, m0, 2 vextracti32x4 xm3, m0, 3 movq [dstq+strideq*0], xm0 movq [dstq+strideq*2], xm1 movq [r10 +strideq*0], xm2 movq [r10 +strideq*2], xm3 movhps [dstq+strideq*1], xm0 movhps [dstq+r9 ], xm1 movhps [r10 +strideq*1], xm2 movhps [r10 +r9 ], xm3 RET .sec_only: movifnidn t1d, secm call .sec .end_no_clip: mova xm8, [base+end_perm] kxnorb k1, k1, k1 vpshldd m4, m0, 8 ; (px << 8) + ((sum > -8) << 4) vpshldd m5, m1, 8 vpshldd m6, m2, 8 vpshldd m7, m3, 8 paddw m0, m4 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) paddw m1, m5 paddw m2, m6 paddw m3, m7 vpermb m0, m8, m0 vpermb m1, m8, m1 vpermb m2, m8, m2 vpermb m3, m8, m3 add r10, 2 punpckldq m4, m0, m1 punpckhdq m0, m1 punpckldq m5, m2, m3 punpckhdq m2, m3 movq [dstq+strideq*0], xm4 movq [dstq+strideq*2], xm0 movq [r10 +strideq*0], xm5 movq [r10 +strideq*2], xm2 movhps [dstq+strideq*1], xm4 movhps [dstq+r9 ], xm0 movhps [r10 +strideq*1], xm5 movhps [r10 +r9 ], xm2 RET .mask_edges_sec_only: movifnidn t1d, secm call .mask_edges_sec jmp .end_no_clip ALIGN function_align .mask_edges: mov t0d, r6d mov t1d, r6d or t0d, 0xA ; top-left 4x4 has bottom and right or t1d, 0x9 ; top-right 4x4 has bottom and left vpbroadcastq m26, [base+edge_mask+t0*8] vpbroadcastq m27, [base+edge_mask+t1*8] mov t1d, r6d or r6d, 0x6 ; bottom-left 4x4 has top and right or t1d, 0x5 ; bottom-right 4x4 has top and left vpbroadcastq m28, [base+edge_mask+r6*8] vpbroadcastq m29, [base+edge_mask+t1*8] mov t0d, dirm test prid, prid jz .mask_edges_sec_only vpaddd m20, m30, [base+cdef_dirs+(t0+2)*4] {1to16} vpshufbitqmb k1, m26, m20 ; index in-range vpshufbitqmb k2, m27, m20 vpshufbitqmb k3, m28, m20 vpshufbitqmb k4, m29, m20 mova m8, m4 mova m9, m5 mova m10, m6 mova m11, m7 vpermb m8{k1}, m20, m12 vpermb m9{k2}, m20, m13 vpermb m10{k3}, m20, m14 vpermb m11{k4}, m20, m15 mova [rsp+0x00], m26 mova [rsp+0x40], m27 mova [rsp+0x80], m28 mova [rsp+0xC0], m29 CDEF_FILTER_8x8_PRI test t1d, t1d jz .end_no_clip mova m26, [rsp+0x00] mova m27, [rsp+0x40] mova m28, [rsp+0x80] mova m29, [rsp+0xC0] call .mask_edges_sec jmp .end_clip .mask_edges_sec: vpaddd m20, m30, [base+cdef_dirs+(t0+4)*4] {1to16} vpaddd m21, m30, [base+cdef_dirs+(t0+0)*4] {1to16} vpshufbitqmb k1, m26, m20 vpshufbitqmb k2, m27, m20 vpshufbitqmb k3, m28, m20 vpshufbitqmb k4, m29, m20 mova m16, m4 mova m17, m5 mova m18, m6 mova m19, m7 vpermb m16{k1}, m20, m12 vpermb m17{k2}, m20, m13 vpermb m18{k3}, m20, m14 vpermb m19{k4}, m20, m15 vpshufbitqmb k1, m26, m21 vpshufbitqmb k2, m27, m21 vpshufbitqmb k3, m28, m21 vpshufbitqmb k4, m29, m21 vpermb m12, m21, m12 vpermb m13, m21, m13 vpermb m14, m21, m14 vpermb m15, m21, m15 vpblendmb m12{k1}, m4, m12 vpblendmb m13{k2}, m5, m13 vpblendmb m14{k3}, m6, m14 vpblendmb m15{k4}, m7, m15 jmp .sec_main ALIGN function_align .sec: vpaddd m20, m30, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 vpaddd m21, m30, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 vpermb m16, m20, m12 ; pNtl k0s0 k0s1 k1s0 k1s1 vpermb m17, m20, m13 ; pNtr vpermb m18, m20, m14 ; pNbl vpermb m19, m20, m15 ; pNbr vpermb m12, m21, m12 ; pNtl k0s2 k0s3 k1s2 k1s3 vpermb m13, m21, m13 ; pNtr vpermb m14, m21, m14 ; pNbl vpermb m15, m21, m15 ; pNbr .sec_main: %macro CDEF_FILTER_8x8_SEC 4-5 0 ; load constants vpcmpub k1, m4, %1, 6 vpcmpub k2, m5, %2, 6 vpcmpub k3, m6, %3, 6 vpcmpub k4, m7, %4, 6 psubb m20, %1, m4 psubb m21, %2, m5 psubb m22, %3, m6 psubb m23, %4, m7 %if %5 vpbroadcastb m28, t1d lzcnt t1d, t1d vpbroadcastq m29, [r3+t1*8] %endif vpsubb m20{k1}, m4, %1 vpsubb m21{k2}, m5, %2 vpsubb m22{k3}, m6, %3 vpsubb m23{k4}, m7, %4 gf2p8affineqb m24, m20, m29, 0 gf2p8affineqb m25, m21, m29, 0 gf2p8affineqb m26, m22, m29, 0 gf2p8affineqb m27, m23, m29, 0 %if %5 vpbroadcastd m30, [base+sec_tap] %endif psubusb m24, m28, m24 psubusb m25, m28, m25 psubusb m26, m28, m26 psubusb m27, m28, m27 pminub m20, m24 pminub m21, m25 pminub m22, m26 pminub m23, m27 mova m24, m30 mova m25, m30 mova m26, m30 mova m27, m30 vpsubb m24{k1}, m31, m30 vpsubb m25{k2}, m31, m30 vpsubb m26{k3}, m31, m30 vpsubb m27{k4}, m31, m30 vpdpbusd m0, m20, m24 vpdpbusd m1, m21, m25 vpdpbusd m2, m22, m26 vpdpbusd m3, m23, m27 %endmacro CDEF_FILTER_8x8_SEC m16, m17, m18, m19, 1 CDEF_FILTER_8x8_SEC m12, m13, m14, m15 ret %endif ; HAVE_AVX512ICL && ARCH_X86_64