ref: 8ff894639bbfaf849f54424f0d8241b076f0fd97
parent: e79e5ceb2cb74fc466e2868c4725d98ccca7cac7
author: Ronald S. Bultje <rsbultje@gmail.com>
date: Sat Dec 21 11:06:45 EST 2019
SSSE3 implementations of film grain gen_grain_y_ar0_8bpc_c: 84853.3 gen_grain_y_ar0_8bpc_ssse3: 23528.0 gen_grain_y_ar1_8bpc_c: 140775.5 gen_grain_y_ar1_8bpc_ssse3: 70410.2 gen_grain_y_ar2_8bpc_c: 251311.3 gen_grain_y_ar2_8bpc_ssse3: 95222.2 gen_grain_y_ar3_8bpc_c: 394763.0 gen_grain_y_ar3_8bpc_ssse3: 103541.9 gen_grain_uv_ar0_8bpc_420_c: 29773.7 gen_grain_uv_ar0_8bpc_420_ssse3: 7068.9 gen_grain_uv_ar1_8bpc_420_c: 46113.2 gen_grain_uv_ar1_8bpc_420_ssse3: 22148.1 gen_grain_uv_ar2_8bpc_420_c: 70061.4 gen_grain_uv_ar2_8bpc_420_ssse3: 25479.0 gen_grain_uv_ar3_8bpc_420_c: 113826.0 gen_grain_uv_ar3_8bpc_420_ssse3: 30004.9 fguv_32x32xn_8bpc_420_csfl0_c: 8148.9 fguv_32x32xn_8bpc_420_csfl0_ssse3: 1371.3 fguv_32x32xn_8bpc_420_csfl1_c: 6391.9 fguv_32x32xn_8bpc_420_csfl1_ssse3: 1034.8 fgy_32x32xn_8bpc_c: 14201.3 fgy_32x32xn_8bpc_ssse3: 3443.0
--- a/include/dav1d/headers.h
+++ b/include/dav1d/headers.h
@@ -318,7 +318,7 @@
int scaling_shift;
int ar_coeff_lag;
int8_t ar_coeffs_y[24];
- int8_t ar_coeffs_uv[2][25];
+ int8_t ar_coeffs_uv[2][25 + 3 /* padding for alignment purposes */];
uint64_t ar_coeff_shift;
int grain_scale_shift;
int uv_mult[2];
--- a/src/meson.build
+++ b/src/meson.build
@@ -151,6 +151,7 @@
'x86/looprestoration.asm',
'x86/mc.asm',
'x86/cdef_sse.asm',
+ 'x86/film_grain_ssse3.asm',
'x86/ipred_ssse3.asm',
'x86/itx_ssse3.asm',
'x86/loopfilter_ssse3.asm',
--- a/src/x86/film_grain.asm
+++ b/src/x86/film_grain.asm
@@ -469,7 +469,7 @@
.ar0:
INIT_YMM avx2
DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift
- imul uvd, 25
+ imul uvd, 28
mov shiftd, [fg_dataq+FGData.ar_coeff_shift]
movd xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq]
movd xm3, [base+hmul_bits+shiftq*2]
@@ -539,7 +539,7 @@
.ar1:
INIT_XMM avx2
DEFINE_ARGS buf, bufy, fg_data, uv, val3, cf3, min, max, x, shift
- imul uvd, 25
+ imul uvd, 28
mov shiftd, [fg_dataq+FGData.ar_coeff_shift]
movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3]
movd xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq]
@@ -606,7 +606,7 @@
.ar2:
DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift
mov shiftd, [fg_dataq+FGData.ar_coeff_shift]
- imul uvd, 25
+ imul uvd, 28
vpbroadcastw xm15, [base+round_vals-12+shiftq*2]
pmovsxbw xm8, [fg_dataq+FGData.ar_coeffs_uv+uvq+0] ; cf0-7
pmovsxbw xm9, [fg_dataq+FGData.ar_coeffs_uv+uvq+8] ; cf8-12
@@ -696,7 +696,7 @@
%assign stack_size_padded (stack_size_padded+16*12)
%assign stack_size (stack_size+16*12)
mov shiftd, [fg_dataq+FGData.ar_coeff_shift]
- imul uvd, 25
+ imul uvd, 28
vpbroadcastw xm14, [base+round_vals-12+shiftq*2]
pmovsxbw xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] ; cf0-7
pmovsxbw xm1, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 8] ; cf8-15
--- a/src/x86/film_grain_init_tmpl.c
+++ b/src/x86/film_grain_init_tmpl.c
@@ -28,6 +28,11 @@
#include "src/cpu.h"
#include "src/film_grain.h"
+decl_generate_grain_y_fn(dav1d_generate_grain_y_ssse3);
+decl_generate_grain_uv_fn(dav1d_generate_grain_uv_420_ssse3);
+decl_fgy_32x32xn_fn(dav1d_fgy_32x32xn_ssse3);
+decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i420_ssse3);
+
decl_generate_grain_y_fn(dav1d_generate_grain_y_avx2);
decl_generate_grain_uv_fn(dav1d_generate_grain_uv_420_avx2);
decl_fgy_32x32xn_fn(dav1d_fgy_32x32xn_avx2);
@@ -35,6 +40,15 @@
COLD void bitfn(dav1d_film_grain_dsp_init_x86)(Dav1dFilmGrainDSPContext *const c) {const unsigned flags = dav1d_get_cpu_flags();
+
+ if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return;
+
+#if BITDEPTH == 8
+ c->generate_grain_y = dav1d_generate_grain_y_ssse3;
+ c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_generate_grain_uv_420_ssse3;
+ c->fgy_32x32xn = dav1d_fgy_32x32xn_ssse3;
+ c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_fguv_32x32xn_i420_ssse3;
+#endif
if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return;
--- /dev/null
+++ b/src/x86/film_grain_ssse3.asm
@@ -1,0 +1,2938 @@
+; Copyright © 2019, VideoLAN and dav1d authors
+; Copyright © 2019, Two Orioles, LLC
+; All rights reserved.
+;
+; Redistribution and use in source and binary forms, with or without
+; modification, are permitted provided that the following conditions are met:
+;
+; 1. Redistributions of source code must retain the above copyright notice, this
+; list of conditions and the following disclaimer.
+;
+; 2. Redistributions in binary form must reproduce the above copyright notice,
+; this list of conditions and the following disclaimer in the documentation
+; and/or other materials provided with the distribution.
+;
+; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+%include "ext/x86/x86inc.asm"
+
+SECTION_RODATA
+
+pw_1024: times 8 dw 1024
+pb_27_17: times 8 db 27, 17
+pb_17_27: times 8 db 17, 27
+pb_mask: db 0, 0x80, 0x80, 0, 0x80, 0, 0, 0x80, 0x80, 0, 0, 0x80, 0, 0x80, 0x80, 0
+rnd_next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058
+byte_blend: db 0, 0, 0, 0xff, 0, 0, 0, 0
+pw_seed_xor: times 2 dw 0xb524
+ times 2 dw 0x49d8
+pb_23_22: times 2 db 23, 22
+pb_1: times 4 db 1
+hmul_bits: dw 32768, 16384, 8192, 4096
+round: dw 2048, 1024, 512
+mul_bits: dw 256, 128, 64, 32, 16
+round_vals: dw 32, 64, 128, 256, 512
+max: dw 255, 240, 235
+min: dw 0, 16
+pw_1: dw 1
+
+%define pb_27_17_17_27 pb_17_27 - 2
+
+%macro JMP_TABLE 1-*
+ %xdefine %1_table %%table
+ %xdefine %%base %1_table
+ %xdefine %%prefix mangle(private_prefix %+ _%1)
+ %%table:
+ %rep %0 - 1
+ dd %%prefix %+ .ar%2 - %%base
+ %rotate 1
+ %endrep
+%endmacro
+
+JMP_TABLE generate_grain_y_ssse3, 0, 1, 2, 3
+JMP_TABLE generate_grain_uv_420_ssse3, 0, 1, 2, 3
+
+struc FGData
+ .seed: resd 1
+ .num_y_points: resd 1
+ .y_points: resb 14 * 2
+ .chroma_scaling_from_luma: resd 1
+ .num_uv_points: resd 2
+ .uv_points: resb 2 * 10 * 2
+ .scaling_shift: resd 1
+ .ar_coeff_lag: resd 1
+ .ar_coeffs_y: resb 24
+ .ar_coeffs_uv: resb 2 * 28 ; includes padding
+ .ar_coeff_shift: resq 1
+ .grain_scale_shift: resd 1
+ .uv_mult: resd 2
+ .uv_luma_mult: resd 2
+ .uv_offset: resd 2
+ .overlap_flag: resd 1
+ .clip_to_restricted_range: resd 1
+endstruc
+
+cextern gaussian_sequence
+
+SECTION .text
+
+%macro SCRATCH 3
+%if ARCH_X86_32
+ mova [rsp+%3*mmsize], m%1
+%define m%2 [rsp+%3*mmsize]
+%else
+ SWAP %1, %2
+%endif
+%endmacro
+
+INIT_XMM ssse3
+cglobal generate_grain_y, 2, 7 + 2 * ARCH_X86_64, 16, buf, fg_data
+ LEA r4, $$
+%define base r4-$$
+ movq m1, [base+rnd_next_upperbit_mask]
+ movq m4, [base+mul_bits]
+ movq m7, [base+hmul_bits]
+ mov r2d, [fg_dataq+FGData.grain_scale_shift]
+ movd m2, [base+round+r2*2]
+ movd m0, [fg_dataq+FGData.seed]
+ mova m5, [base+pb_mask]
+ pshuflw m2, m2, q0000
+ pshuflw m0, m0, q0000
+ mov r2, -73*82
+ sub bufq, r2
+ lea r3, [base+gaussian_sequence]
+.loop:
+ pand m6, m0, m1
+ psrlw m3, m6, 10
+ por m6, m3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set
+ pmullw m6, m4 ; bits 0x0f00 are set
+ pshufb m3, m5, m6 ; set 15th bit for next 4 seeds
+ psllq m6, m3, 30
+ por m3, m6
+ psllq m6, m3, 15
+ por m3, m6 ; aggregate each bit into next seed's high bit
+ pmulhuw m6, m0, m7
+ por m3, m6 ; 4 next output seeds
+ pshuflw m0, m3, q3333
+ psrlw m3, 5
+%if ARCH_X86_64
+ movq r6, m3
+ mov r8, r6
+ movzx r5d, r6w
+ shr r6d, 16
+ shr r8, 32
+ movzx r7, r8w
+ shr r8, 16
+
+ movd m6, [r3+r5*2]
+ pinsrw m6, [r3+r6*2], 1
+ pinsrw m6, [r3+r7*2], 2
+ pinsrw m6, [r3+r8*2], 3
+%else
+ movd r6, m3
+ pshuflw m3, m3, q3232
+ movzx r5, r6w
+ shr r6, 16
+
+ movd m6, [r3+r5*2]
+ pinsrw m6, [r3+r6*2], 1
+
+ movd r6, m3
+ movzx r5, r6w
+ shr r6, 16
+
+ pinsrw m6, [r3+r5*2], 2
+ pinsrw m6, [r3+r6*2], 3
+%endif
+ pmulhrsw m6, m2
+ packsswb m6, m6
+ movd [bufq+r2], m6
+ add r2, 4
+ jl .loop
+
+ ; auto-regression code
+ movsxd r2, [fg_dataq+FGData.ar_coeff_lag]
+ movsxd r2, [base+generate_grain_y_ssse3_table+r2*4]
+ lea r2, [r2+base+generate_grain_y_ssse3_table]
+ jmp r2
+
+.ar1:
+%if ARCH_X86_32
+ DEFINE_ARGS buf, fg_data, cf3, unused, val3, min, max
+%elif WIN64
+ DEFINE_ARGS shift, fg_data, cf3, buf, val3, min, max, x, val0
+ mov bufq, r0
+%else
+ DEFINE_ARGS buf, fg_data, cf3, shift, val3, min, max, x, val0
+%endif
+ movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_y+3]
+ movd m4, [fg_dataq+FGData.ar_coeffs_y]
+ mov ecx, [fg_dataq+FGData.ar_coeff_shift]
+%if ARCH_X86_32
+ mov r1m, cf3d
+ DEFINE_ARGS buf, shift, val3, min, max, x, val0
+%define hd r0mp
+%define cf3d r1mp
+%elif WIN64
+ DEFINE_ARGS shift, h, cf3, buf, val3, min, max, x, val0
+%else
+ DEFINE_ARGS buf, h, cf3, shift, val3, min, max, x, val0
+%endif
+ pxor m6, m6
+ pcmpgtb m7, m6, m4
+ punpcklbw m4, m7
+ pinsrw m4, [base+pw_1], 3
+ pshufd m5, m4, q1111
+ pshufd m4, m4, q0000
+ movd m3, [base+round_vals+shiftq*2-12] ; rnd
+ pshuflw m3, m3, q0000
+ sub bufq, 82*73-(82*3+79)
+ mov hd, 70
+ mov mind, -128
+ mov maxd, 127
+.y_loop_ar1:
+ mov xq, -76
+ movsx val3d, byte [bufq+xq-1]
+.x_loop_ar1:
+ movq m0, [bufq+xq-82-1] ; top/left
+ pcmpgtb m7, m6, m0
+ punpcklbw m0, m7
+ psrldq m2, m0, 2 ; top
+ psrldq m1, m0, 4 ; top/right
+ punpcklwd m0, m2
+ punpcklwd m1, m3
+ pmaddwd m0, m4
+ pmaddwd m1, m5
+ paddd m0, m1
+.x_loop_ar1_inner:
+ movd val0d, m0
+ psrldq m0, 4
+ imul val3d, cf3d
+ add val3d, val0d
+ sar val3d, shiftb
+ movsx val0d, byte [bufq+xq]
+ add val3d, val0d
+ cmp val3d, maxd
+ cmovg val3d, maxd
+ cmp val3d, mind
+ cmovl val3d, mind
+ mov byte [bufq+xq], val3b
+ ; keep val3d in-place as left for next x iteration
+ inc xq
+ jz .x_loop_ar1_end
+ test xq, 3
+ jnz .x_loop_ar1_inner
+ jmp .x_loop_ar1
+
+.x_loop_ar1_end:
+ add bufq, 82
+ dec hd
+ jg .y_loop_ar1
+.ar0:
+ RET
+
+.ar2:
+%if ARCH_X86_32
+%assign stack_offset_old stack_offset
+ ALLOC_STACK -16*8
+%endif
+ DEFINE_ARGS buf, fg_data, shift
+ mov shiftd, [fg_dataq+FGData.ar_coeff_shift]
+ movd m6, [base+round_vals-12+shiftq*2]
+ movd m7, [base+byte_blend+1]
+ SCRATCH 7, 15, 7
+ movq m0, [fg_dataq+FGData.ar_coeffs_y+0] ; cf0-7
+ movd m1, [fg_dataq+FGData.ar_coeffs_y+8] ; cf8-11
+ pxor m7, m7
+ pshuflw m6, m6, q0000
+ punpcklwd m6, m7
+ pcmpgtb m4, m7, m0
+ pcmpgtb m5, m7, m1
+ punpcklbw m0, m4
+ punpcklbw m1, m5
+ DEFINE_ARGS buf, fg_data, h, x
+ pshufd m4, m1, q0000
+ pshufd m5, m1, q1111
+ pshufd m3, m0, q3333
+ pshufd m2, m0, q2222
+ pshufd m1, m0, q1111
+ pshufd m0, m0, q0000
+ SCRATCH 0, 8, 0
+ SCRATCH 1, 9, 1
+ SCRATCH 2, 10, 2
+ SCRATCH 3, 11, 3
+ SCRATCH 4, 12, 4
+ SCRATCH 5, 13, 5
+ SCRATCH 6, 14, 6
+ sub bufq, 82*73-(82*3+79)
+ mov hd, 70
+.y_loop_ar2:
+ mov xq, -76
+
+.x_loop_ar2:
+ movq m0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5]
+ movhps m0, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5]
+ pcmpgtb m2, m7, m0
+ punpckhbw m1, m0, m2
+ punpcklbw m0, m2
+ psrldq m5, m0, 2 ; y=-2,x=[-1,+5]
+ psrldq m3, m1, 2 ; y=-1,x=[-1,+5]
+ psrldq m4, m1, 4 ; y=-1,x=[+0,+5]
+ punpcklwd m2, m0, m5
+ punpcklwd m3, m4
+ pmaddwd m2, m8
+ pmaddwd m3, m11
+ paddd m2, m3
+
+ psrldq m4, m0, 4 ; y=-2,x=[+0,+5]
+ psrldq m5, m0, 6 ; y=-2,x=[+1,+5]
+ psrldq m6, m0, 8 ; y=-2,x=[+2,+5]
+ punpcklwd m4, m5
+ punpcklwd m6, m1
+ psrldq m5, m1, 6 ; y=-1,x=[+1,+5]
+ psrldq m1, m1, 8 ; y=-1,x=[+2,+5]
+ punpcklwd m5, m1
+ pmaddwd m4, m9
+ pmaddwd m6, m10
+ pmaddwd m5, m12
+ paddd m4, m6
+ paddd m2, m5
+ paddd m2, m4
+ paddd m2, m14
+
+ movq m0, [bufq+xq-2] ; y=0,x=[-2,+5]
+.x_loop_ar2_inner:
+ pcmpgtb m4, m7, m0
+ punpcklbw m1, m0, m4
+ pmaddwd m3, m1, m13
+ paddd m3, m2
+ psrldq m1, 4 ; y=0,x=0
+ psrldq m2, 4 ; shift top to next pixel
+ psrad m3, [fg_dataq+FGData.ar_coeff_shift]
+ ; don't packssdw since we only care about one value
+ paddw m3, m1
+ packsswb m3, m3
+ pslldq m3, 2
+ pand m3, m15
+ pandn m1, m15, m0
+ por m0, m1, m3
+ psrldq m0, 1
+ ; overwrite 2 pixels, but that's ok
+ movd [bufq+xq-1], m0
+ inc xq
+ jz .x_loop_ar2_end
+ test xq, 3
+ jnz .x_loop_ar2_inner
+ jmp .x_loop_ar2
+
+.x_loop_ar2_end:
+ add bufq, 82
+ dec hd
+ jg .y_loop_ar2
+ RET
+
+.ar3:
+ DEFINE_ARGS buf, fg_data, shift
+%if ARCH_X86_32
+%assign stack_offset stack_offset_old
+ ALLOC_STACK -16*14
+%elif WIN64
+ SUB rsp, 16*6
+%assign stack_size_padded (stack_size_padded+16*6)
+%assign stack_size (stack_size+16*6)
+%else
+ ALLOC_STACK -16*6
+%endif
+ mov shiftd, [fg_dataq+FGData.ar_coeff_shift]
+ movd m6, [base+round_vals-12+shiftq*2]
+ movd m7, [base+byte_blend]
+ movu m0, [fg_dataq+FGData.ar_coeffs_y+ 0] ; cf0-15
+ movq m2, [fg_dataq+FGData.ar_coeffs_y+16] ; cf16-23
+ pxor m3, m3
+ pcmpgtb m4, m3, m0
+ pcmpgtb m3, m2
+ pshuflw m6, m6, q0000
+ SCRATCH 6, 14, 12
+ SCRATCH 7, 15, 13
+ punpckhbw m1, m0, m4
+ punpcklbw m0, m4
+ punpcklbw m2, m3
+ pshufd m3, m0, q1111
+ pshufd m4, m0, q2222
+ pshufd m5, m0, q3333
+ pshufd m0, m0, q0000
+ mova [rsp+ 0*16], m0
+ mova [rsp+ 1*16], m3
+ mova [rsp+ 2*16], m4
+ mova [rsp+ 3*16], m5
+ pshufd m6, m1, q1111
+ pshufd m7, m1, q2222
+ pshufd m5, m1, q3333
+ pshufd m1, m1, q0000
+ pshufd m3, m2, q1111
+ psrldq m0, m2, 10
+ pinsrw m2, [base+pw_1], 5
+ pshufd m4, m2, q2222
+ pshufd m2, m2, q0000
+ pinsrw m0, [base+round_vals+shiftq*2-10], 3
+ mova [rsp+ 4*16], m1
+ mova [rsp+ 5*16], m6
+ SCRATCH 7, 8, 6
+ SCRATCH 5, 9, 7
+ SCRATCH 2, 10, 8
+ SCRATCH 3, 11, 9
+ SCRATCH 4, 12, 10
+ SCRATCH 0, 13, 11
+ DEFINE_ARGS buf, fg_data, h, x
+ sub bufq, 82*73-(82*3+79)
+ mov hd, 70
+.y_loop_ar3:
+ mov xq, -76
+
+.x_loop_ar3:
+ movu m0, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12]
+ pxor m3, m3
+ pcmpgtb m3, m0
+ punpckhbw m2, m0, m3
+ punpcklbw m0, m3
+
+ psrldq m5, m0, 2
+ psrldq m6, m0, 4
+ psrldq m7, m0, 6
+ punpcklwd m4, m0, m5
+ punpcklwd m6, m7
+ pmaddwd m4, [rsp+ 0*16]
+ pmaddwd m6, [rsp+ 1*16]
+ paddd m4, m6
+
+ movu m1, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12]
+ pxor m5, m5
+ pcmpgtb m5, m1
+ punpckhbw m3, m1, m5
+ punpcklbw m1, m5
+ palignr m6, m2, m0, 10
+ palignr m7, m2, m0, 12
+ psrldq m0, 8
+ punpcklwd m0, m6
+ punpcklwd m7, m1
+ pmaddwd m0, [rsp+ 2*16]
+ pmaddwd m7, [rsp+ 3*16]
+ paddd m0, m7
+ paddd m0, m4
+
+ psrldq m4, m1, 2
+ psrldq m5, m1, 4
+ psrldq m6, m1, 6
+ psrldq m7, m1, 8
+ punpcklwd m4, m5
+ punpcklwd m6, m7
+ pmaddwd m4, [rsp+ 4*16]
+ pmaddwd m6, [rsp+ 5*16]
+ paddd m4, m6
+ paddd m0, m4
+
+ movu m2, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12]
+ pxor m7, m7
+ pcmpgtb m7, m2
+ punpckhbw m5, m2, m7
+ punpcklbw m2, m7
+ palignr m7, m3, m1, 10
+ palignr m3, m1, 12
+ psrldq m1, m2, 2
+ punpcklwd m7, m3
+ punpcklwd m3, m2, m1
+ pmaddwd m7, m8
+ pmaddwd m3, m9
+ paddd m7, m3
+ paddd m0, m7
+
+ psrldq m6, m2, 4
+ psrldq m1, m2, 6
+ psrldq m3, m2, 8
+ palignr m4, m5, m2, 10
+ palignr m5, m5, m2, 12
+
+ punpcklwd m6, m1
+ punpcklwd m3, m4
+ punpcklwd m5, m14
+ pmaddwd m6, m10
+ pmaddwd m3, m11
+ pmaddwd m5, m12
+ paddd m0, m6
+ paddd m3, m5
+ paddd m0, m3
+
+ movq m1, [bufq+xq-3] ; y=0,x=[-3,+4]
+.x_loop_ar3_inner:
+ pxor m5, m5
+ pcmpgtb m5, m1
+ punpcklbw m2, m1, m5
+ pmaddwd m2, m13
+ pshufd m3, m2, q1111
+ paddd m2, m3 ; left+cur
+ paddd m2, m0 ; add top
+ psrldq m0, 4
+ psrad m2, [fg_dataq+FGData.ar_coeff_shift]
+ ; don't packssdw since we only care about one value
+ packsswb m2, m2
+ pslldq m2, 3
+ pand m2, m15
+ pandn m3, m15, m1
+ por m1, m2, m3
+ movd [bufq+xq-3], m1
+ psrldq m1, 1
+ inc xq
+ jz .x_loop_ar3_end
+ test xq, 3
+ jnz .x_loop_ar3_inner
+ jmp .x_loop_ar3
+
+.x_loop_ar3_end:
+ add bufq, 82
+ dec hd
+ jg .y_loop_ar3
+ RET
+
+INIT_XMM ssse3
+cglobal generate_grain_uv_420, 1, 7 + 3 * ARCH_X86_64, 16, buf, bufy, fg_data, uv
+ movifnidn r2, r2mp
+ movifnidn r3, r3mp
+ LEA r4, $$
+%define base r4-$$
+ movq m1, [base+rnd_next_upperbit_mask]
+ movq m4, [base+mul_bits]
+ movq m7, [base+hmul_bits]
+ mov r5d, [fg_dataq+FGData.grain_scale_shift]
+ movd m6, [base+round+r5*2]
+ mova m5, [base+pb_mask]
+ movd m0, [fg_dataq+FGData.seed]
+ movd m2, [base+pw_seed_xor+uvq*4]
+ pxor m0, m2
+ pshuflw m6, m6, q0000
+ pshuflw m0, m0, q0000
+ lea r6, [base+gaussian_sequence]
+%if ARCH_X86_64
+ mov r7d, 38
+%else
+ mov r3mp, 38
+%endif
+ add bufq, 44
+.loop_y:
+ mov r5, -44
+.loop_x:
+ pand m2, m0, m1
+ psrlw m3, m2, 10
+ por m2, m3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set
+ pmullw m2, m4 ; bits 0x0f00 are set
+ pshufb m3, m5, m2 ; set 15th bit for next 4 seeds
+ psllq m2, m3, 30
+ por m3, m2
+ psllq m2, m3, 15
+ por m3, m2 ; aggregate each bit into next seed's high bit
+ pmulhuw m2, m0, m7
+ por m2, m3 ; 4 next output seeds
+ pshuflw m0, m2, q3333
+ psrlw m2, 5
+%if ARCH_X86_64
+ movd r9d, m2
+ pshuflw m2, m2, q3232
+ movzx r8, r9w
+ shr r9, 16
+
+ movd m3, [r6+r8*2]
+ pinsrw m3, [r6+r9*2], 1
+
+ movd r9d, m2
+ movzx r8, r9w
+ shr r9, 16
+
+ pinsrw m3, [r6+r8*2], 2
+ pinsrw m3, [r6+r9*2], 3
+%else
+ movd r2, m2
+ pshuflw m2, m2, q3232
+ movzx r1, r2w
+ shr r2, 16
+
+ movd m3, [r6+r1*2]
+ pinsrw m3, [r6+r2*2], 1
+
+ movd r2, m2
+ movzx r1, r2w
+ shr r2, 16
+
+ pinsrw m3, [r6+r1*2], 2
+ pinsrw m3, [r6+r2*2], 3
+%endif
+ pmulhrsw m3, m6
+ packsswb m3, m3
+ movd [bufq+r5], m3
+ add r5, 4
+ jl .loop_x
+ add bufq, 82
+%if ARCH_X86_64
+ dec r7d
+%else
+ dec r3mp
+%endif
+ jg .loop_y
+
+%if ARCH_X86_32
+ mov r2, r2mp
+%endif
+
+ ; auto-regression code
+ movsxd r5, [fg_dataq+FGData.ar_coeff_lag]
+ movsxd r5, [base+generate_grain_uv_420_ssse3_table+r5*4]
+ lea r5, [r5+base+generate_grain_uv_420_ssse3_table]
+ jmp r5
+
+.ar0:
+ DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift
+ movifnidn bufyq, bufymp
+%if ARCH_X86_32
+%assign stack_offset_old stack_offset
+ ALLOC_STACK -2*16
+%endif
+ imul uvd, 28
+ mov shiftd, [fg_dataq+FGData.ar_coeff_shift]
+ movd m5, [fg_dataq+FGData.ar_coeffs_uv+uvq]
+ movd m4, [base+hmul_bits+shiftq*2]
+ movd m1, [base+byte_blend]
+ DEFINE_ARGS buf, bufy, h
+ pxor m0, m0
+ pcmpgtb m0, m5
+ punpcklbw m5, m0
+ movd m7, [base+pb_1]
+ movd m6, [base+hmul_bits+4]
+ pshuflw m5, m5, q0000
+ pshuflw m4, m4, q0000
+ pshufd m7, m7, q0000
+ pshuflw m6, m6, q0000
+ punpcklqdq m5, m5
+ punpcklqdq m4, m4
+ punpcklqdq m6, m6
+ punpcklbw m1, m1
+ SCRATCH 1, 8, 0
+ SCRATCH 4, 9, 1
+ sub bufq, 82*38+82-(82*3+41)
+ add bufyq, 3+82*3
+ mov hd, 35
+.y_loop_ar0:
+ ; first 32 pixels
+ movu m1, [bufyq]
+ movu m2, [bufyq+82]
+ movu m3, [bufyq+16]
+ movu m4, [bufyq+82+16]
+ pmaddubsw m0, m7, m1
+ pmaddubsw m1, m7, m2
+ pmaddubsw m2, m7, m3
+ pmaddubsw m3, m7, m4
+ paddw m0, m1
+ paddw m2, m3
+ pmulhrsw m0, m6
+ pmulhrsw m2, m6
+ pmullw m0, m5
+ pmullw m2, m5
+ pmulhrsw m0, m9
+ pmulhrsw m2, m9
+ packsswb m0, m2
+ movu m1, [bufq]
+ punpckhbw m2, m0, m1
+ punpcklbw m0, m1
+ pmaddubsw m1, m7, m2
+ pmaddubsw m2, m7, m0
+ packsswb m2, m1
+ movu [bufq], m2
+ add bufyq, 32
+ add bufq, 16
+ xor hd, 0x10000
+ test hd, 0x10000
+ jnz .y_loop_ar0
+
+ ; last 6 pixels
+ movu m1, [bufyq]
+ movu m2, [bufyq+82]
+ pmaddubsw m0, m7, m1
+ pmaddubsw m1, m7, m2
+ paddw m0, m1
+ pmulhrsw m0, m6
+ pmullw m0, m5
+ pmulhrsw m0, m9
+ packsswb m0, m0
+ movq m1, [bufq]
+ punpcklbw m0, m1
+ pmaddubsw m2, m7, m0
+ packsswb m2, m2
+ pandn m0, m8, m2
+ pand m1, m8
+ por m0, m1
+ movq [bufq], m0
+
+ add bufq, 82-32
+ add bufyq, 82*2-64
+ dec hd
+ jg .y_loop_ar0
+ RET
+
+.ar1:
+%if ARCH_X86_32
+%assign stack_offset stack_offset_old
+%assign stack_size_padded 0
+%xdefine rstk rsp
+%endif
+ DEFINE_ARGS buf, bufy, fg_data, uv, val3, cf3, min, max, x
+ imul uvd, 28
+ movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3]
+ movd m4, [fg_dataq+FGData.ar_coeffs_uv+uvq-1]
+ pinsrw m4, [fg_dataq+FGData.ar_coeffs_uv+uvq+4], 2
+%if ARCH_X86_32
+ mov r3mp, cf3d
+ DEFINE_ARGS buf, shift, fg_data, val3, min, max, x
+%elif WIN64
+ DEFINE_ARGS shift, bufy, fg_data, buf, val3, cf3, min, max, x
+ mov bufq, r0
+%else
+ DEFINE_ARGS buf, bufy, fg_data, shift, val3, cf3, min, max, x
+%endif
+ mov shiftd, [fg_dataq+FGData.ar_coeff_shift]
+ movd m3, [base+round_vals+shiftq*2-12] ; rnd
+ movd m7, [base+pb_1]
+ movd m6, [base+hmul_bits+4]
+ psrldq m4, 1
+%if ARCH_X86_32
+ DEFINE_ARGS buf, shift, val0, val3, min, max, x
+%elif WIN64
+ DEFINE_ARGS shift, bufy, h, buf, val3, cf3, min, max, x, val0
+%else
+ DEFINE_ARGS buf, bufy, h, shift, val3, cf3, min, max, x, val0
+%endif
+ pxor m5, m5
+ punpcklwd m3, m5
+ punpcklwd m6, m6
+ pcmpgtb m5, m4
+ punpcklbw m4, m5
+ pshufd m5, m4, q1111
+ pshufd m4, m4, q0000
+ pshufd m3, m3, q0000
+ pshufd m7, m7, q0000
+ pshufd m6, m6, q0000
+ sub bufq, 82*38+44-(82*3+41)
+%if ARCH_X86_32
+ add r1mp, 79+82*3
+ mov r0mp, 35
+%else
+ add bufyq, 79+82*3
+ mov hd, 35
+%endif
+ mov mind, -128
+ mov maxd, 127
+.y_loop_ar1:
+ mov xq, -38
+ movsx val3d, byte [bufq+xq-1]
+.x_loop_ar1:
+%if ARCH_X86_32
+ mov r2, r1mp
+ movq m0, [r2+xq*2]
+ movq m1, [r2+xq*2+82]
+%else
+ movq m0, [bufyq+xq*2]
+ movq m1, [bufyq+xq*2+82]
+%endif
+ pmaddubsw m2, m7, m0
+ pmaddubsw m0, m7, m1
+ paddw m2, m0
+ pmulhrsw m2, m6
+
+ movq m0, [bufq+xq-82-1] ; top/left
+ pxor m1, m1
+ pcmpgtb m1, m0
+ punpcklbw m0, m1
+ psrldq m1, m0, 4 ; top/right
+ punpcklwd m1, m2
+ psrldq m2, m0, 2 ; top
+ punpcklwd m0, m2
+ pmaddwd m0, m4
+ pmaddwd m1, m5
+ paddd m0, m1
+ paddd m0, m3
+.x_loop_ar1_inner:
+ movd val0d, m0
+ psrldq m0, 4
+%if ARCH_X86_32
+ imul val3d, r3mp
+%else
+ imul val3d, cf3d
+%endif
+ add val3d, val0d
+ sar val3d, shiftb
+ movsx val0d, byte [bufq+xq]
+ add val3d, val0d
+ cmp val3d, maxd
+ cmovg val3d, maxd
+ cmp val3d, mind
+ cmovl val3d, mind
+ mov byte [bufq+xq], val3b
+ ; keep val3d in-place as left for next x iteration
+ inc xq
+ jz .x_loop_ar1_end
+ test xq, 3
+ jnz .x_loop_ar1_inner
+ jmp .x_loop_ar1
+
+.x_loop_ar1_end:
+ add bufq, 82
+%if ARCH_X86_32
+ add r1mp, 82*2
+ dec r0mp
+%else
+ add bufyq, 82*2
+ dec hd
+%endif
+ jg .y_loop_ar1
+ RET
+
+.ar2:
+%if ARCH_X86_32
+%assign stack_offset stack_offset_old
+%assign stack_size_padded 0
+%xdefine rstk rsp
+ ALLOC_STACK -8*16
+%endif
+ DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift
+ movifnidn bufyq, bufymp
+ mov shiftd, [fg_dataq+FGData.ar_coeff_shift]
+ imul uvd, 28
+ movd m7, [base+round_vals-12+shiftq*2]
+ movu m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+0] ; cf0-12
+ pxor m2, m2
+ pcmpgtb m2, m0
+ punpckhbw m1, m0, m2
+ punpcklbw m0, m2
+ pinsrw m1, [base+pw_1], 5
+ punpcklwd m7, m7
+ pshufd m7, m7, q0000
+ DEFINE_ARGS buf, bufy, fg_data, h, unused, x
+ pshufd m4, m1, q0000
+ pshufd m5, m1, q1111
+ pshufd m6, m1, q2222
+ pshufd m3, m0, q3333
+ pshufd m2, m0, q2222
+ pshufd m1, m0, q1111
+ pshufd m0, m0, q0000
+ SCRATCH 0, 8, 0
+ SCRATCH 1, 9, 1
+ SCRATCH 2, 10, 2
+ SCRATCH 3, 11, 3
+ SCRATCH 4, 12, 4
+ SCRATCH 5, 13, 5
+ SCRATCH 6, 14, 6
+ SCRATCH 7, 15, 7
+ movd m7, [base+hmul_bits+4]
+ movd m6, [base+pb_1]
+ punpcklwd m7, m7
+ pshufd m6, m6, q0000
+ pshufd m7, m7, q0000
+ sub bufq, 82*38+44-(82*3+41)
+ add bufyq, 79+82*3
+ mov hd, 35
+.y_loop_ar2:
+ mov xq, -38
+
+.x_loop_ar2:
+ pxor m2, m2
+ movq m0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5]
+ movhps m0, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5]
+ pcmpgtb m2, m0
+ punpckhbw m1, m0, m2
+ punpcklbw m0, m2
+ psrldq m5, m0, 2 ; y=-2,x=[-1,+5]
+ psrldq m3, m1, 2 ; y=-1,x=[-1,+5]
+ psrldq m4, m1, 4 ; y=-1,x=[+0,+5]
+ punpcklwd m2, m0, m5
+ punpcklwd m3, m4
+ pmaddwd m2, m8
+ pmaddwd m3, m11
+ paddd m2, m3
+
+ psrldq m4, m0, 4 ; y=-2,x=[+0,+5]
+ psrldq m5, m0, 6 ; y=-2,x=[+1,+5]
+ psrldq m0, 8 ; y=-2,x=[+2,+5]
+ punpcklwd m4, m5
+ punpcklwd m0, m1
+ psrldq m3, m1, 6 ; y=-1,x=[+1,+5]
+ psrldq m1, m1, 8 ; y=-1,x=[+2,+5]
+ punpcklwd m3, m1
+ pmaddwd m4, m9
+ pmaddwd m0, m10
+ pmaddwd m3, m12
+ paddd m4, m0
+ paddd m2, m3
+ paddd m2, m4
+
+ movq m0, [bufyq+xq*2]
+ movq m3, [bufyq+xq*2+82]
+ pmaddubsw m1, m6, m0
+ pmaddubsw m0, m6, m3
+ paddw m0, m1
+ pmulhrsw m0, m7
+ punpcklwd m0, m15
+ pmaddwd m0, m14
+ paddd m2, m0
+
+ movq m0, [bufq+xq-2] ; y=0,x=[-2,+5]
+ pxor m4, m4
+ movd m5, [base+byte_blend+1]
+ punpcklbw m5, m5
+.x_loop_ar2_inner:
+ pcmpgtb m1, m4, m0
+ punpcklbw m0, m1
+ pmaddwd m3, m0, m13
+ paddd m3, m2
+ psrldq m2, 4 ; shift top to next pixel
+ psrad m3, [fg_dataq+FGData.ar_coeff_shift]
+ pslldq m3, 4
+ pand m3, m5
+ paddw m0, m3
+ packsswb m0, m0
+ movd [bufq+xq-2], m0
+ psrldq m0, 1
+ inc xq
+ jz .x_loop_ar2_end
+ test xq, 3
+ jnz .x_loop_ar2_inner
+ jmp .x_loop_ar2
+
+.x_loop_ar2_end:
+ add bufq, 82
+ add bufyq, 82*2
+ dec hd
+ jg .y_loop_ar2
+ RET
+
+.ar3:
+%if ARCH_X86_32
+%assign stack_offset stack_offset_old
+%assign stack_size_padded 0
+%xdefine rstk rsp
+%endif
+ DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift
+ movifnidn bufyq, bufymp
+%if ARCH_X86_32
+ ALLOC_STACK -15*16
+%else
+ SUB rsp, 16*7
+%assign stack_size_padded (stack_size_padded+16*7)
+%assign stack_size (stack_size+16*7)
+%endif
+ mov shiftd, [fg_dataq+FGData.ar_coeff_shift]
+ imul uvd, 28
+
+ movu m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] ; cf0-15
+ pxor m3, m3
+ pcmpgtb m3, m0
+ punpckhbw m1, m0, m3
+ punpcklbw m0, m3
+ pshufd m2, m0, q1111
+ pshufd m3, m0, q2222
+ pshufd m4, m0, q3333
+ pshufd m0, m0, q0000
+ pshufd m5, m1, q1111
+ pshufd m6, m1, q2222
+ pshufd m7, m1, q3333
+ pshufd m1, m1, q0000
+ mova [rsp+ 0*16], m0
+ mova [rsp+ 1*16], m2
+ mova [rsp+ 2*16], m3
+ mova [rsp+ 3*16], m4
+ mova [rsp+ 4*16], m1
+ mova [rsp+ 5*16], m5
+ mova [rsp+ 6*16], m6
+ SCRATCH 7, 8, 7
+
+ movu m2, [fg_dataq+FGData.ar_coeffs_uv+uvq+16] ; cf16-24 [24=luma]
+ pxor m4, m4
+ pcmpgtb m4, m2
+ punpckhbw m5, m2, m4
+ punpcklbw m2, m4
+ pshufd m4, m2, q3232
+ punpcklwd m3, m4, m5
+ pshuflw m5, m4, q3321
+ pshufd m4, m3, q0000
+ pshufd m3, m2, q1111
+ pshufd m2, m2, q0000
+ pinsrw m5, [base+round_vals+shiftq*2-10], 3
+ SCRATCH 2, 9, 8
+ SCRATCH 3, 10, 9
+ SCRATCH 4, 11, 10
+ SCRATCH 5, 12, 11
+
+ movd m2, [base+round_vals-12+shiftq*2]
+ movd m1, [base+pb_1]
+ movd m3, [base+hmul_bits+4]
+ pxor m0, m0
+ punpcklwd m2, m0
+ punpcklwd m3, m3
+ pshufd m2, m2, q0000
+ pshufd m1, m1, q0000
+ pshufd m3, m3, q0000
+ SCRATCH 1, 13, 12
+ SCRATCH 2, 14, 13
+ SCRATCH 3, 15, 14
+
+ DEFINE_ARGS buf, bufy, fg_data, h, unused, x
+ sub bufq, 82*38+44-(82*3+41)
+ add bufyq, 79+82*3
+ mov hd, 35
+.y_loop_ar3:
+ mov xq, -38
+
+.x_loop_ar3:
+ movu m0, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12]
+ pxor m4, m4
+ pcmpgtb m4, m0
+ punpckhbw m3, m0, m4
+ punpcklbw m0, m4
+
+ psrldq m5, m0, 2
+ psrldq m6, m0, 4
+ psrldq m7, m0, 6
+ punpcklwd m4, m0, m5
+ punpcklwd m6, m7
+ pmaddwd m4, [rsp+ 0*16]
+ pmaddwd m6, [rsp+ 1*16]
+ paddd m4, m6
+
+ palignr m2, m3, m0, 10
+ palignr m3, m0, 12
+ psrldq m0, 8
+
+ movu m1, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12]
+ pxor m6, m6
+ pcmpgtb m6, m1
+ punpckhbw m5, m1, m6
+ punpcklbw m1, m6
+
+ punpcklwd m0, m2
+ punpcklwd m3, m1
+ pmaddwd m0, [rsp+ 2*16]
+ pmaddwd m3, [rsp+ 3*16]
+ paddd m0, m3
+ paddd m0, m4
+
+ movu m2, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12]
+ pxor m7, m7
+ pcmpgtb m7, m2
+ punpckhbw m6, m2, m7
+ punpcklbw m2, m7
+
+ palignr m3, m5, m1, 10
+ palignr m5, m1, 12
+ psrldq m4, m2, 2
+
+ punpcklwd m3, m5
+ punpcklwd m5, m2, m4
+ pmaddwd m3, [rsp+ 6*16]
+ pmaddwd m5, m8
+ paddd m3, m5
+ paddd m0, m3
+
+ psrldq m3, m1, 2
+ psrldq m4, m1, 4
+ psrldq m5, m1, 6
+ psrldq m1, 8
+
+ punpcklwd m3, m4
+ punpcklwd m5, m1
+ pmaddwd m3, [rsp+ 4*16]
+ pmaddwd m5, [rsp+ 5*16]
+ paddd m3, m5
+ paddd m0, m3
+
+ movq m1, [bufyq+xq*2]
+ movq m3, [bufyq+xq*2+82]
+ pmaddubsw m5, m13, m1
+ pmaddubsw m7, m13, m3
+ paddw m7, m5
+ pmulhrsw m7, m15
+
+ psrldq m1, m2, 4
+ psrldq m3, m2, 6
+ palignr m4, m6, m2, 10
+ palignr m6, m2, 12
+ psrldq m2, 8
+
+ punpcklwd m1, m3
+ punpcklwd m2, m4
+ punpcklwd m6, m7
+ pmaddwd m1, m9
+ pmaddwd m2, m10
+ pmaddwd m6, m11
+ paddd m1, m2
+ paddd m0, m6
+ paddd m0, m1
+ paddd m0, m14
+
+ movq m1, [bufq+xq-3] ; y=0,x=[-3,+4]
+ pxor m4, m4
+ movd m5, [base+byte_blend]
+.x_loop_ar3_inner:
+ pcmpgtb m2, m4, m1
+ punpcklbw m3, m1, m2
+ pmaddwd m2, m3, m12
+ pshufd m3, m2, q1111
+ paddd m2, m3 ; left+cur
+ paddd m2, m0 ; add top
+ psrldq m0, 4
+ psrad m2, [fg_dataq+FGData.ar_coeff_shift]
+ ; don't packssdw, we only care about one value
+ packsswb m2, m2
+ pandn m3, m5, m1
+ pslld m2, 24
+ pand m2, m5
+ por m1, m2, m3
+ movd [bufq+xq-3], m1
+ psrldq m1, 1
+ inc xq
+ jz .x_loop_ar3_end
+ test xq, 3
+ jnz .x_loop_ar3_inner
+ jmp .x_loop_ar3
+
+.x_loop_ar3_end:
+ add bufq, 82
+ add bufyq, 82*2
+ dec hd
+ jg .y_loop_ar3
+ RET
+
+%macro vpgatherdw 5-6 ; dst, src, base, tmp_gpr[x2], tmp_xmm_reg
+%assign %%idx 0
+%define %%tmp %2
+%if %0 == 6
+%define %%tmp %6
+%endif
+%rep 4
+%if %%idx == 0
+ movd %5 %+ d, %2
+ pshuflw %%tmp, %2, q3232
+%else
+ movd %5 %+ d, %%tmp
+%if %%idx == 2
+ punpckhqdq %%tmp, %%tmp
+%elif %%idx == 4
+ psrlq %%tmp, 32
+%endif
+%endif
+ movzx %4 %+ d, %5 %+ w
+ shr %5 %+ d, 16
+
+%if %%idx == 0
+ movd %1, [%3+%4]
+%else
+ pinsrw %1, [%3+%4], %%idx + 0
+%endif
+ pinsrw %1, [%3+%5], %%idx + 1
+%assign %%idx %%idx+2
+%endrep
+%endmacro
+
+INIT_XMM ssse3
+; fgy_32x32xn(dst, src, stride, fg_data, w, scaling, grain_lut, h, sby)
+%if ARCH_X86_32
+%if STACK_ALIGNMENT < mmsize
+cglobal fgy_32x32xn, 0, 7, 16, 0 - (6 * mmsize + (9 + 3) * gprsize), \
+ dst, src, scaling, unused1, fg_data, picptr, unused2
+ ; copy stack arguments to new position post-alignment, so that we
+ ; don't have to keep the old stack location in a separate register
+ mov r0, r0m
+ mov r1, r2m
+ mov r2, r4m
+ mov r3, r6m
+ mov r4, r7m
+ mov r5, r8m
+
+ mov [rsp+6*mmsize+ 3*gprsize], r0
+ mov [rsp+6*mmsize+ 5*gprsize], r1
+ mov [rsp+6*mmsize+ 7*gprsize], r2
+ mov [rsp+6*mmsize+ 9*gprsize], r3
+ mov [rsp+6*mmsize+10*gprsize], r4
+ mov [rsp+6*mmsize+11*gprsize], r5
+%else
+cglobal fgy_32x32xn, 0, 7, 16, 6 * mmsize + (3 + 1) * gprsize, \
+ dst, src, scaling, unused1, fg_data, picptr, unused2
+%endif
+ mov srcq, srcm
+ mov fg_dataq, r3m
+ mov scalingq, r5m
+%if STACK_ALIGNMENT < mmsize
+%define r0m [rsp+6*mmsize+ 3*gprsize]
+%define r1m [rsp+6*mmsize+ 4*gprsize]
+%define r2m [rsp+6*mmsize+ 5*gprsize]
+%define r3m [rsp+6*mmsize+ 6*gprsize]
+%define r4m [rsp+6*mmsize+ 7*gprsize]
+%define r5m [rsp+6*mmsize+ 8*gprsize]
+%define r6m [rsp+6*mmsize+ 9*gprsize]
+%define r7m [rsp+6*mmsize+10*gprsize]
+%define r8m [rsp+6*mmsize+11*gprsize]
+%endif
+ LEA r5, pb_mask
+%define base r5-pb_mask
+ mov r5m, picptrq
+%else
+cglobal fgy_32x32xn, 6, 15, 16, dst, src, stride, fg_data, w, scaling, grain_lut
+ lea r7, [pb_mask]
+%define base r7-pb_mask
+%endif
+ mov r6d, [fg_dataq+FGData.scaling_shift]
+ movd m3, [base+mul_bits+r6*2-14]
+ mov r6d, [fg_dataq+FGData.clip_to_restricted_range]
+ pcmpeqw m2, m2
+ psrldq m2, 14
+ movd m4, [base+max+r6*4]
+ movd m5, [base+min+r6*2]
+ punpcklwd m3, m3
+ punpcklwd m4, m4
+ punpcklwd m5, m5
+ pshufd m3, m3, q0000
+ pshufd m4, m4, q0000
+ pshufd m5, m5, q0000
+ SCRATCH 2, 10, 0
+ SCRATCH 3, 11, 1
+ SCRATCH 4, 12, 2
+ SCRATCH 5, 13, 3
+
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap
+%else
+ DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap
+%endif
+
+ mov sbyd, r8m
+ mov overlapd, [fg_dataq+FGData.overlap_flag] ; left_overlap: overlap & 1
+ test overlapd, overlapd
+ jz .no_vertical_overlap
+ mova m6, [base+pw_1024]
+ movd m7, [base+pb_27_17_17_27]
+ SCRATCH 6, 14, 4
+ SCRATCH 7, 15, 5
+ test sbyd, sbyd
+ jnz .vertical_overlap
+ ; fall-through
+
+.no_vertical_overlap:
+ mov r8m, overlapd
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, see, fg_data, picptr, unused
+ imul seed, (173 << 24) | 37
+%else
+ imul seed, sbyd, (173 << 24) | 37
+%endif
+ add seed, (105 << 24) | 178
+ rol seed, 8
+ movzx seed, seew
+ xor seed, [fg_dataq+FGData.seed]
+
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak
+
+ mov r3m, seed
+ mov wq, r4m
+%else
+ DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
+ unused1, unused2, see, unused3
+%endif
+
+ lea src_bakq, [srcq+wq]
+ neg wq
+ sub dstmp, srcq
+%if ARCH_X86_32
+ mov r1m, src_bakq
+ mov r4m, wq
+ DEFINE_ARGS dst, src, scaling, see, unused1, unused2, unused3
+%endif
+
+.loop_x:
+%if ARCH_X86_32
+ mov seed, r3m
+%endif
+ mov r6d, seed
+ or seed, 0xEFF4
+ shr r6d, 1
+ test seeb, seeh
+ lea seed, [r6+0x8000]
+ cmovp seed, r6d ; updated seed
+%if ARCH_X86_32
+ mov r3m, seed
+
+ DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx
+
+ mov offxd, offyd
+%else
+ DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
+ offx, offy, see, unused
+
+ mov offyd, seed
+ mov offxd, seed
+%endif
+ ror offyd, 8
+ shr offxd, 12
+ and offyd, 0xf
+ imul offyd, 164
+ lea offyq, [offyq+offxq*2+747] ; offy*stride+offx
+
+%if ARCH_X86_32
+ ; r0m=dst, r1m=src_bak, r2m=stride, r3m=see, r4m=w, r5m=picptr,
+ ; r6m=grain_lut, r7m=h, r8m=overlap_v|h
+ DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
+%else
+ DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
+ h, offxy, see, unused
+%endif
+
+.loop_x_odd:
+ mov hd, r7m
+ mov grain_lutq, grain_lutmp
+.loop_y:
+ ; src
+ mova m0, [srcq]
+ pxor m2, m2
+ punpckhbw m1, m0, m2
+ punpcklbw m0, m2 ; m0-1: src as word
+
+ ; scaling[src]
+%if ARCH_X86_32
+ vpgatherdw m4, m0, scalingq, r0, r5, m3
+ vpgatherdw m5, m1, scalingq, r0, r5, m3
+%else
+ vpgatherdw m4, m0, scalingq, r12, r13, m3
+ vpgatherdw m5, m1, scalingq, r12, r13, m3
+%endif
+ pcmpeqw m3, m3
+ psrlw m3, 8
+ pand m4, m3
+ pand m5, m3
+
+ ; grain = grain_lut[offy+y][offx+x]
+ movu m3, [grain_lutq+offxyq]
+ pcmpgtb m7, m2, m3
+ punpcklbw m2, m3, m7
+ punpckhbw m3, m7
+
+ ; noise = round2(scaling[src] * grain, scaling_shift)
+ pmullw m2, m4
+ pmullw m3, m5
+ pmulhrsw m2, m11
+ pmulhrsw m3, m11
+
+ ; dst = clip_pixel(src, noise)
+ paddw m0, m2
+ paddw m1, m3
+ pmaxsw m0, m13
+ pmaxsw m1, m13
+ pminsw m0, m12
+ pminsw m1, m12
+ packuswb m0, m1
+ movifnidn dstq, dstmp
+ mova [dstq+srcq], m0
+
+ add srcq, r2mp
+ add grain_lutq, 82
+ dec hd
+ jg .loop_y
+
+%if ARCH_X86_32
+ add r4mp, 16
+%else
+ add wq, 16
+%endif
+ jge .end
+%if ARCH_X86_32
+ mov srcq, r1mp
+ add srcq, r4mp
+ xor r8mp, 4
+ test r8mp, 4
+%else
+ lea srcq, [src_bakq+wq]
+ test srcq, 16 ; this relies on buffer alignment...
+%endif
+ jz .next_blk
+
+ add offxyd, 16
+ test dword r8m, 2 ; r8m & 2 = have_top_overlap
+ jz .loop_x_odd
+
+%if ARCH_X86_32
+ add dword [rsp+6*mmsize+1*gprsize], 16
+%else
+ add r11d, 16 ; top_offxyd
+%endif
+ jnz .loop_x_odd_v_overlap
+
+.next_blk:
+ test dword r8m, 1
+ jz .loop_x
+
+ test dword r8m, 2
+ jnz .loop_x_hv_overlap
+
+ ; horizontal overlap (without vertical overlap)
+.loop_x_h_overlap:
+%if ARCH_X86_32
+ ; r0m=dst, r1m=src_bak, r2m=stride, r3m=see, r4m=w, r5m=picptr,
+ ; r6m=grain_lut, r7m=h, r8m=overlap_v|h
+ DEFINE_ARGS dst, src, scaling, offxy, unused1, unused2, unused3
+
+ add offxyd, 16 ; left_offxyd
+ mov [rsp+6*mmsize+0*gprsize], offxyd
+
+ DEFINE_ARGS dst, src, scaling, see, unused1, unused2, unused3
+
+ mov seed, r3m
+%else
+ DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
+ offx, offy, see, left_offxy
+
+ lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx
+%endif
+
+ mov r6d, seed
+ or seed, 0xEFF4
+ shr r6d, 1
+ test seeb, seeh
+ lea seed, [r6+0x8000]
+ cmovp seed, r6d ; updated seed
+
+%if ARCH_X86_32
+ mov r3m, seed
+
+ DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx
+
+ mov offxd, offyd
+%else
+ mov offyd, seed
+ mov offxd, seed
+%endif
+ ror offyd, 8
+ shr offxd, 12
+ and offyd, 0xf
+ imul offyd, 164
+ lea offyq, [offyq+offxq*2+747] ; offy*stride+offx
+
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
+%else
+ DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
+ h, offxy, see, left_offxy
+%endif
+
+ mov hd, r7m
+ mov grain_lutq, grain_lutmp
+.loop_y_h_overlap:
+ ; src
+ mova m0, [srcq]
+ pxor m2, m2
+ punpckhbw m1, m0, m2
+ punpcklbw m0, m2 ; m0-1: src as word
+
+ ; scaling[src]
+%if ARCH_X86_32
+ vpgatherdw m4, m0, scalingq, r0, r5, m3
+ vpgatherdw m5, m1, scalingq, r0, r5, m3
+%else
+ vpgatherdw m4, m0, scalingq, r12, r13, m3
+ vpgatherdw m5, m1, scalingq, r12, r13, m3
+%endif
+ pcmpeqw m3, m3
+ psrlw m3, 8
+ pand m4, m3
+ pand m5, m3
+
+ ; grain = grain_lut[offy+y][offx+x]
+ movu m3, [grain_lutq+offxyq]
+%if ARCH_X86_32
+ mov r5, [rsp+6*mmsize+0*gprsize]
+ movd m7, [grain_lutq+r5]
+%else
+ movd m7, [grain_lutq+left_offxyq]
+%endif
+ punpcklbw m7, m3
+ pmaddubsw m6, m15, m7
+ pmulhrsw m6, m14
+ packsswb m6, m6
+ pand m6, m10
+ pandn m7, m10, m3
+ por m6, m7
+ pcmpgtb m2, m6
+ punpcklbw m7, m6, m2
+ punpckhbw m6, m2
+
+ ; noise = round2(scaling[src] * grain, scaling_shift)
+ pmullw m7, m4
+ pmullw m6, m5
+ pmulhrsw m7, m11
+ pmulhrsw m6, m11
+
+ ; dst = clip_pixel(src, noise)
+ paddw m0, m7
+ paddw m1, m6
+ pmaxsw m0, m13
+ pmaxsw m1, m13
+ pminsw m0, m12
+ pminsw m1, m12
+ packuswb m0, m1
+ movifnidn dstq, dstmp
+ mova [dstq+srcq], m0
+
+ add srcq, r2mp
+ add grain_lutq, 82
+ dec hd
+ jg .loop_y_h_overlap
+
+%if ARCH_X86_32
+ add r4mp, 16
+%else
+ add wq, 16
+%endif
+ jge .end
+%if ARCH_X86_32
+ mov srcq, r1m
+ add srcq, r4m
+ xor r8mp, 4
+%else
+ lea srcq, [src_bakq+wq]
+%endif
+ ; assert(srcq & 16) != 0
+ add offxyd, 16
+
+ ; since this half-block had left-overlap, the next does not
+ test dword r8m, 2 ; have_top_overlap
+ jz .loop_x_odd
+%if ARCH_X86_32
+ add dword [rsp+6*mmsize+1*gprsize], 16
+%else
+ add r11d, 16 ; top_offxyd
+%endif
+ jmp .loop_x_odd_v_overlap
+
+.end:
+ RET
+
+.vertical_overlap:
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap
+%else
+ DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, tmp, sby, see, overlap
+%endif
+
+ or overlapd, 2 ; top_overlap: overlap & 2
+ mov r8m, overlapd
+ movzx sbyd, sbyb
+%if ARCH_X86_32
+ imul r4, [fg_dataq+FGData.seed], 0x00010001
+ DEFINE_ARGS tmp, src, scaling, sby, see, picptr, unused
+%else
+ imul seed, [fg_dataq+FGData.seed], 0x00010001
+%endif
+ imul tmpd, sbyd, 173 * 0x00010001
+ imul sbyd, 37 * 0x01000100
+ add tmpd, (105 << 16) | 188
+ add sbyd, (178 << 24) | (141 << 8)
+ and tmpd, 0x00ff00ff
+ and sbyd, 0xff00ff00
+ xor seed, tmpd
+%if ARCH_X86_32
+ xor sbyd, seed ; (cur_seed << 16) | top_seed
+
+ DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak
+
+ mov r3m, seed
+ mov wq, r4m
+%else
+ xor seed, sbyd ; (cur_seed << 16) | top_seed
+
+ DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
+ tmp, unused2, see, unused3
+%endif
+
+ lea src_bakq, [srcq+wq]
+ neg wq
+ sub dstmp, srcq
+%if ARCH_X86_32
+ mov r1m, src_bakq
+ mov r4m, wq
+ DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2
+%endif
+
+.loop_x_v_overlap:
+%if ARCH_X86_32
+ mov seed, r3m
+%endif
+ ; we assume from the block above that bits 8-15 of tmpd are zero'ed,
+ ; because of the 'and tmpd, 0x00ff00ff' above
+ mov r6d, seed
+ or seed, 0xeff4eff4
+ test seeb, seeh
+ setp tmpb ; parity of top_seed
+ shr seed, 16
+ shl tmpd, 16
+ test seeb, seeh
+ setp tmpb ; parity of cur_seed
+ or r6d, 0x00010001
+ xor tmpd, r6d
+ mov seed, tmpd
+ ror seed, 1 ; updated (cur_seed << 16) | top_seed
+
+%if ARCH_X86_32
+ mov r3m, seed
+
+ DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx
+
+ mov offxd, offyd
+%else
+ DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
+ offx, offy, see, unused, top_offxy
+
+ mov offyd, seed
+ mov offxd, seed
+%endif
+
+ ror offyd, 8
+ ror offxd, 12
+ and offyd, 0xf000f
+ and offxd, 0xf000f
+ imul offyd, 164
+ ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
+ lea offyq, [offyq+offxq*2+0x10001*747+32*82]
+
+%if ARCH_X86_32
+ DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut
+%else
+ DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
+ h, offxy, see, unused, top_offxy
+%endif
+
+ movzx top_offxyd, offxyw
+%if ARCH_X86_32
+ mov [rsp+6*mmsize+1*gprsize], top_offxyd
+
+ DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
+%endif
+ shr offxyd, 16
+
+.loop_x_odd_v_overlap:
+%if ARCH_X86_32
+ mov r5, r5m
+ lea r5, [base+pb_27_17]
+ mov [rsp+5*mmsize+8], r5
+%else
+ mova m8, [pb_27_17]
+%endif
+ mov hd, r7m
+ mov grain_lutq, grain_lutmp
+.loop_y_v_overlap:
+ ; src
+ mova m0, [srcq]
+ pxor m2, m2
+ punpckhbw m1, m0, m2
+ punpcklbw m0, m2 ; m0-1: src as word
+
+ ; scaling[src]
+%if ARCH_X86_32
+ vpgatherdw m4, m0, scalingq, r0, r5, m3
+ vpgatherdw m5, m1, scalingq, r0, r5, m3
+%else
+ vpgatherdw m4, m0, scalingq, r12, r13, m3
+ vpgatherdw m5, m1, scalingq, r12, r13, m3
+%endif
+ pcmpeqw m3, m3
+ psrlw m3, 8
+ pand m4, m3
+ pand m5, m3
+
+ ; grain = grain_lut[offy+y][offx+x]
+ movu m3, [grain_lutq+offxyq]
+%if ARCH_X86_32
+ mov r5, [rsp+6*mmsize+1*gprsize]
+ movu m7, [grain_lutq+r5]
+%else
+ movu m7, [grain_lutq+top_offxyq]
+%endif
+ punpckhbw m6, m7, m3
+ punpcklbw m7, m3
+%if ARCH_X86_32
+ mov r5, [rsp+5*mmsize+8]
+ pmaddubsw m3, [r5], m6
+ pmaddubsw m6, [r5], m7
+%else
+ pmaddubsw m3, m8, m6
+ pmaddubsw m6, m8, m7
+%endif
+ pmulhrsw m3, m14
+ pmulhrsw m6, m14
+ packsswb m6, m3
+ pcmpgtb m7, m2, m6
+ punpcklbw m2, m6, m7
+ punpckhbw m6, m7
+
+ ; noise = round2(scaling[src] * grain, scaling_shift)
+ pmullw m2, m4
+ pmullw m6, m5
+ pmulhrsw m2, m11
+ pmulhrsw m6, m11
+
+ ; dst = clip_pixel(src, noise)
+ paddw m0, m2
+ paddw m1, m6
+ pmaxsw m0, m13
+ pmaxsw m1, m13
+ pminsw m0, m12
+ pminsw m1, m12
+ packuswb m0, m1
+ movifnidn dstq, dstmp
+ mova [dstq+srcq], m0
+
+%if ARCH_X86_32
+ add dword [rsp+5*mmsize+8], mmsize
+%else
+ mova m8, [pb_17_27]
+%endif
+ add srcq, r2mp
+ add grain_lutq, 82
+ dec hw
+ jz .end_y_v_overlap
+ ; 2 lines get vertical overlap, then fall back to non-overlap code for
+ ; remaining (up to) 30 lines
+ xor hd, 0x10000
+ test hd, 0x10000
+ jnz .loop_y_v_overlap
+ jmp .loop_y
+
+.end_y_v_overlap:
+%if ARCH_X86_32
+ add r4mp, 16
+%else
+ add wq, 16
+%endif
+ jge .end_hv
+%if ARCH_X86_32
+ mov srcq, r1mp
+ add srcq, r4mp
+ xor r8mp, 4
+ test r8mp, 4
+%else
+ lea srcq, [src_bakq+wq]
+ test srcq, 16
+%endif
+ jz .loop_x_hv_overlap
+ add offxyd, 16
+%if ARCH_X86_32
+ add dword [rsp+6*mmsize+1*gprsize], 16
+%else
+ add top_offxyd, 16
+%endif
+ jmp .loop_x_odd_v_overlap
+
+.loop_x_hv_overlap:
+%if ARCH_X86_32
+ mov r5, r5m
+ lea r5, [base+pb_27_17]
+ mov [rsp+5*mmsize+8], r5
+
+ DEFINE_ARGS tmp, src, scaling, offxy, w, picptr, src_bak
+
+ mov r5, [rsp+6*mmsize+1*gprsize]
+ mov r4, offxyd
+ add r5, 16
+ add r4, 16
+ mov [rsp+6*mmsize+2*gprsize], r5 ; topleft_offxy
+ mov [rsp+6*mmsize+0*gprsize], r4 ; left_offxy
+
+ DEFINE_ARGS tmp, src, scaling, see, w, picptr, src_bak
+
+ xor tmpd, tmpd
+ mov seed, r3m
+%else
+ mova m8, [pb_27_17]
+
+ DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
+ tmp, unused2, see, unused3
+
+ ; we assume from the block above that bits 8-15 of tmpd are zero'ed
+%endif
+ mov r6d, seed
+ or seed, 0xeff4eff4
+ test seeb, seeh
+ setp tmpb ; parity of top_seed
+ shr seed, 16
+ shl tmpd, 16
+ test seeb, seeh
+ setp tmpb ; parity of cur_seed
+ or r6d, 0x00010001
+ xor tmpd, r6d
+ mov seed, tmpd
+ ror seed, 1 ; updated (cur_seed << 16) | top_seed
+
+%if ARCH_X86_32
+ mov r3m, seed
+
+ DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx
+
+ mov offxd, offyd
+%else
+ DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
+ offx, offy, see, left_offxy, top_offxy, topleft_offxy
+
+ lea topleft_offxyq, [top_offxyq+16]
+ lea left_offxyq, [offyq+16]
+ mov offyd, seed
+ mov offxd, seed
+%endif
+ ror offyd, 8
+ ror offxd, 12
+ and offyd, 0xf000f
+ and offxd, 0xf000f
+ imul offyd, 164
+ ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
+ lea offyq, [offyq+offxq*2+0x10001*747+32*82]
+
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
+
+ movzx r5, offxyw ; top_offxy
+ mov [rsp+6*mmsize+1*gprsize], r5
+%else
+ DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \
+ h, offxy, see, left_offxy, top_offxy, topleft_offxy
+
+ movzx top_offxyd, offxyw
+%endif
+ shr offxyd, 16
+
+ mov hd, r7m
+ mov grain_lutq, grain_lutmp
+.loop_y_hv_overlap:
+ ; grain = grain_lut[offy+y][offx+x]
+ movu m3, [grain_lutq+offxyq]
+%if ARCH_X86_32
+ mov r5, [rsp+6*mmsize+1*gprsize] ; top_offxy
+ mov r0, [rsp+6*mmsize+0*gprsize] ; left_offxy
+ movu m6, [grain_lutq+r5]
+ mov r5, [rsp+6*mmsize+2*gprsize] ; topleft_offxy
+ movd m4, [grain_lutq+r0]
+ movd m7, [grain_lutq+r5]
+%else
+ movu m6, [grain_lutq+top_offxyq]
+ movd m4, [grain_lutq+left_offxyq]
+ movd m7, [grain_lutq+topleft_offxyq]
+%endif
+ ; do h interpolation first (so top | top/left -> top, left | cur -> cur)
+ punpcklbw m4, m3
+ punpcklbw m7, m6
+ pmaddubsw m2, m15, m4
+ pmaddubsw m4, m15, m7
+ pmulhrsw m2, m14
+ pmulhrsw m4, m14
+ packsswb m2, m2
+ packsswb m4, m4
+ pand m2, m10
+ pand m4, m10
+ pandn m7, m10, m3
+ pandn m3, m10, m6
+ por m7, m2
+ por m3, m4
+ ; followed by v interpolation (top | cur -> cur)
+ punpckhbw m4, m3, m7
+ punpcklbw m3, m7
+%if ARCH_X86_32
+ mov r5, [rsp+5*mmsize+8]
+ pmaddubsw m7, [r5], m4
+ pmaddubsw m4, [r5], m3
+%else
+ pmaddubsw m7, m8, m4
+ pmaddubsw m4, m8, m3
+%endif
+ pmulhrsw m7, m14
+ pmulhrsw m4, m14
+ packsswb m4, m7
+ pxor m2, m2
+ pcmpgtb m7, m2, m4
+ punpcklbw m3, m4, m7
+ punpckhbw m4, m7
+
+ ; src
+ mova m0, [srcq]
+ punpckhbw m1, m0, m2
+ punpcklbw m0, m2 ; m0-1: src as word
+
+ ; scaling[src]
+%if ARCH_X86_32
+ vpgatherdw m5, m0, scalingq, r0, r5, m7
+ vpgatherdw m6, m1, scalingq, r0, r5, m7
+%else
+ vpgatherdw m5, m0, scalingq, r13, r14, m7
+ vpgatherdw m6, m1, scalingq, r13, r14, m7
+%endif
+ pcmpeqw m7, m7
+ psrlw m7, 8
+ pand m5, m7
+ pand m6, m7
+
+ ; noise = round2(scaling[src] * grain, scaling_shift)
+ pmullw m3, m5
+ pmullw m4, m6
+ pmulhrsw m3, m11
+ pmulhrsw m4, m11
+
+ ; dst = clip_pixel(src, noise)
+ paddw m0, m3
+ paddw m1, m4
+ pmaxsw m0, m13
+ pmaxsw m1, m13
+ pminsw m0, m12
+ pminsw m1, m12
+ packuswb m0, m1
+ movifnidn dstq, dstmp
+ mova [dstq+srcq], m0
+
+%if ARCH_X86_32
+ add dword [rsp+5*mmsize+8], mmsize
+%else
+ mova m8, [pb_17_27]
+%endif
+ add srcq, r2mp
+ add grain_lutq, 82
+ dec hw
+ jz .end_y_hv_overlap
+ ; 2 lines get vertical overlap, then fall back to non-overlap code for
+ ; remaining (up to) 30 lines
+ xor hd, 0x10000
+ test hd, 0x10000
+ jnz .loop_y_hv_overlap
+ jmp .loop_y_h_overlap
+
+.end_y_hv_overlap:
+%if ARCH_X86_32
+ add r4mp, 16
+%else
+ add wq, 16
+%endif
+ jge .end_hv
+%if ARCH_X86_32
+ mov srcq, r1m
+ add srcq, r4m
+ xor r8mp, 4
+%else
+ lea srcq, [src_bakq+wq]
+%endif
+ ; assert(srcq & 16) != 0
+ add offxyd, 16
+%if ARCH_X86_32
+ add dword [rsp+6*mmsize+1*gprsize], 16
+%else
+ add top_offxyd, 16
+%endif
+ jmp .loop_x_odd_v_overlap
+
+.end_hv:
+ RET
+
+INIT_XMM ssse3
+%if ARCH_X86_32
+; fguv_32x32xn_i420_ssse3(dst, src, stride, fg_data, w, scaling, grain_lut, h,
+; sby, luma, lstride, uv_pl, is_id)
+%if STACK_ALIGNMENT < mmsize
+DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8
+cglobal fguv_32x32xn_i420, 0, 7, 8, 0 - (8 * mmsize + (13 + 3) * gprsize), \
+ tmp, src, scaling, h, fg_data, picptr, unused
+ mov r0, r0m
+ mov r1, r2m
+ mov r2, r4m
+ mov r3, r6m
+ mov r4, r7m
+ mov [rsp+8*mmsize+3*gprsize], r0
+ mov [rsp+8*mmsize+5*gprsize], r1
+ mov [rsp+8*mmsize+7*gprsize], r2
+ mov [rsp+8*mmsize+9*gprsize], r3
+ mov [rsp+8*mmsize+10*gprsize], r4
+
+ mov r0, r8m
+ mov r1, r9m
+ mov r2, r10m
+ mov r4, r11m
+ mov r3, r12m
+ mov [rsp+8*mmsize+11*gprsize], r0
+ mov [rsp+8*mmsize+12*gprsize], r1
+ mov [rsp+8*mmsize+13*gprsize], r2
+ mov [rsp+8*mmsize+14*gprsize], r4
+%else
+cglobal fguv_32x32xn_i420, 0, 7, 8, 8 * mmsize + (4) * gprsize, \
+ tmp, src, scaling, h, fg_data, picptr, unused
+%endif
+ mov srcq, srcm
+ mov fg_dataq, r3m
+ mov scalingq, r5m
+%if STACK_ALIGNMENT < mmsize
+%define r0m [rsp+8*mmsize+ 3*gprsize]
+%define r1m [rsp+8*mmsize+ 4*gprsize]
+%define r2m [rsp+8*mmsize+ 5*gprsize]
+%define r3m [rsp+8*mmsize+ 6*gprsize]
+%define r4m [rsp+8*mmsize+ 7*gprsize]
+%define r5m [rsp+8*mmsize+ 8*gprsize]
+%define r6m [rsp+8*mmsize+ 9*gprsize]
+%define r7m [rsp+8*mmsize+10*gprsize]
+%define r8m [rsp+8*mmsize+11*gprsize]
+%define r9m [rsp+8*mmsize+12*gprsize]
+%define r10m [rsp+8*mmsize+13*gprsize]
+%define r11m [rsp+8*mmsize+14*gprsize]
+%define r12m [rsp+8*mmsize+15*gprsize]
+%endif
+ LEA r5, pb_mask
+%define base r5-pb_mask
+ mov r5m, r5
+%else
+cglobal fguv_32x32xn_i420, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \
+ grain_lut, tmp, sby, luma, lstride, uv_pl, is_id
+ lea r8, [pb_mask]
+%define base r8-pb_mask
+%endif
+ mov r6d, [fg_dataq+FGData.scaling_shift]
+ movd m2, [base+byte_blend+3]
+ movd m3, [base+mul_bits+r6*2-14]
+ mov r6d, [fg_dataq+FGData.clip_to_restricted_range]
+ lea tmpd, [r6d*2]
+%if ARCH_X86_32 && STACK_ALIGNMENT < mmsize
+ test r3, r3
+%else
+ cmp dword r12m, 0 ; is_idm
+%endif
+ movd m5, [base+min+r6*2]
+ cmovne r6d, tmpd
+ movd m4, [base+max+r6*2]
+ punpcklwd m3, m3
+ punpcklwd m5, m5
+ punpcklwd m4, m4
+ pshufd m3, m3, q0000
+ pshufd m5, m5, q0000
+ pshufd m4, m4, q0000
+ SCRATCH 2, 10, 0
+ SCRATCH 3, 11, 1
+ SCRATCH 4, 12, 2
+ SCRATCH 5, 13, 3
+
+ cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0
+ jne .csfl
+
+%macro FGUV_32x32xN_LOOP 1 ; not-csfl
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap
+%else
+ DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap
+%endif
+
+%if %1
+ mov r6d, dword r11m
+ movd m0, [fg_dataq+FGData.uv_mult+r6*4]
+ movd m1, [fg_dataq+FGData.uv_luma_mult+r6*4]
+ punpcklbw m6, m1, m0
+ movd m7, [fg_dataq+FGData.uv_offset+r6*4]
+ punpcklwd m6, m6
+ punpcklwd m7, m7
+ pshufd m6, m6, q0000
+ pshufd m7, m7, q0000
+ SCRATCH 6, 14, 4
+ SCRATCH 7, 15, 5
+%endif
+
+ mov sbyd, r8m
+ mov overlapd, [fg_dataq+FGData.overlap_flag] ; left_overlap: overlap & 1
+ test overlapd, overlapd
+ jz %%no_vertical_overlap
+%if ARCH_X86_32
+ movd m1, [base+pb_23_22]
+ mova m0, [base+pw_1024]
+%else
+ movd m1, [pb_23_22]
+ mova m0, [pw_1024]
+%endif
+ pshufd m1, m1, q0000
+ SCRATCH 0, 8, 6
+ SCRATCH 1, 9, 7
+ test sbyd, sbyd
+ jnz %%vertical_overlap
+ ; fall-through
+
+%%no_vertical_overlap:
+ mov r8m, overlapd
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, see, fg_data, picptr, overlap
+ imul seed, (173 << 24) | 37
+%else
+ imul seed, sbyd, (173 << 24) | 37
+%endif
+ add seed, (105 << 24) | 178
+ rol seed, 8
+ movzx seed, seew
+ xor seed, [fg_dataq+FGData.seed]
+
+%if ARCH_X86_32
+ mov r3m, seed
+
+ DEFINE_ARGS luma, src, scaling, see, w, picptr, src_bak
+%define luma_bakq lumaq
+
+ mov wq, r4m
+ shl r10mp, 1
+%else
+ DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
+ unused2, unused3, see, overlap, unused4, src_bak, lstride, luma_bak
+
+ mov lstrideq, r10mp
+%endif
+
+ mov lumaq, r9mp
+ lea src_bakq, [srcq+wq]
+ lea luma_bakq, [lumaq+wq*2]
+ neg wq
+ sub r0mp, srcq
+%if ARCH_X86_32
+ mov r1m, src_bakq
+ mov r11m, luma_bakq
+ mov r4m, wq
+
+ DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2
+%else
+ mov r11mp, src_bakq
+ mov r10mp, strideq
+%endif
+
+%%loop_x:
+%if ARCH_X86_32
+ mov seed, r3m
+%endif
+ mov r6d, seed
+ or seed, 0xEFF4
+ shr r6d, 1
+ test seeb, seeh
+ lea seed, [r6+0x8000]
+ cmovp seed, r6d ; updated seed
+%if ARCH_X86_32
+ mov r3m, seed
+
+ DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx
+
+ mov offxd, offyd
+%else
+ DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
+ offx, offy, see, overlap, unused1, unused2, lstride
+
+ mov offyd, seed
+ mov offxd, seed
+%endif
+ ror offyd, 8
+ shr offxd, 12
+ and offyd, 0xf
+ imul offyd, 82
+ lea offyq, [offyq+offxq+498] ; offy*stride+offx
+
+%if ARCH_X86_32
+ DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut
+%else
+ DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
+ h, offxy, see, overlap, unused1, unused2, lstride, luma_bak
+%endif
+
+ mov hd, r7m
+ mov grain_lutq, grain_lutmp
+%%loop_y:
+ ; src
+%if ARCH_X86_32
+ mov lumaq, r9mp
+%endif
+ mova m4, [lumaq+ 0]
+ mova m6, [lumaq+16]
+ mova m0, [srcq]
+%if ARCH_X86_32
+ add lumaq, r10mp
+ mov r9mp, lumaq
+ mov r5, r5m
+ movd m7, [base+pb_1]
+%else
+ movd m7, [pb_1]
+%endif
+ pshufd m7, m7, q0000
+ pxor m2, m2
+ pmaddubsw m4, m7
+ pmaddubsw m6, m7
+ pavgw m4, m2
+ pavgw m6, m2
+
+%if %1
+ packuswb m4, m6 ; luma
+ punpckhbw m6, m4, m0
+ punpcklbw m4, m0 ; { luma, chroma }+ pmaddubsw m6, m14
+ pmaddubsw m4, m14
+ psraw m6, 6
+ psraw m4, 6
+ paddw m6, m15
+ paddw m4, m15
+ packuswb m4, m6 ; pack+unpack = clip
+ punpckhbw m6, m4, m2
+ punpcklbw m4, m2
+%endif
+
+ ; scaling[luma_src]
+%if ARCH_X86_32
+ vpgatherdw m7, m4, scalingq, r0, r5
+ vpgatherdw m5, m6, scalingq, r0, r5
+%else
+ vpgatherdw m7, m4, scalingq, r12, r2
+ vpgatherdw m5, m6, scalingq, r12, r2
+%endif
+ pcmpeqw m1, m1
+ psrlw m1, 8
+ pand m7, m1
+ pand m5, m1
+
+ ; unpack chroma_source
+ punpckhbw m1, m0, m2
+ punpcklbw m0, m2 ; m0-1: src as word
+
+ ; grain = grain_lut[offy+y][offx+x]
+ movu m3, [grain_lutq+offxyq+ 0]
+ pcmpgtb m6, m2, m3
+ punpcklbw m2, m3, m6
+ punpckhbw m3, m6
+
+ ; noise = round2(scaling[luma_src] * grain, scaling_shift)
+ pmullw m2, m7
+ pmullw m3, m5
+ pmulhrsw m2, m11
+ pmulhrsw m3, m11
+
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
+%endif
+
+ ; dst = clip_pixel(src, noise)
+ paddw m0, m2
+ paddw m1, m3
+ pmaxsw m0, m13
+ pmaxsw m1, m13
+ pminsw m0, m12
+ pminsw m1, m12
+ packuswb m0, m1
+ movifnidn dstq, dstmp
+ mova [dstq+srcq], m0
+
+%if ARCH_X86_32
+ add srcq, r2mp
+ ; we already incremented lumaq above
+%else
+ add srcq, r10mp
+ lea lumaq, [lumaq+lstrideq*2]
+%endif
+ add grain_lutq, 82
+ dec hw
+ jg %%loop_y
+
+%if ARCH_X86_32
+ DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut
+
+ mov wq, r4m
+%endif
+ add wq, 16
+ jge %%end
+%if ARCH_X86_32
+ mov srcq, r1mp
+ mov lumaq, r11mp
+%else
+ mov srcq, r11mp
+%endif
+ lea lumaq, [luma_bakq+wq*2]
+ add srcq, wq
+%if ARCH_X86_32
+ mov r4m, wq
+ mov r9m, lumaq
+%endif
+ test dword r8m, 1
+ jz %%loop_x
+
+ ; r8m = sbym
+ test dword r8m, 2
+ jne %%loop_x_hv_overlap
+
+ ; horizontal overlap (without vertical overlap)
+%%loop_x_h_overlap:
+%if ARCH_X86_32
+ lea r6, [offxyd+16]
+ mov [rsp+8*mmsize+0*gprsize], r6
+
+ DEFINE_ARGS luma, src, scaling, see, w, picptr, grain_lut
+
+ mov seed, r3m
+%else
+ DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
+ offx, offy, see, left_offxy, unused1, unused2, lstride
+
+ lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx
+%endif
+ mov r6d, seed
+ or seed, 0xEFF4
+ shr r6d, 1
+ test seeb, seeh
+ lea seed, [r6+0x8000]
+ cmovp seed, r6d ; updated seed
+
+%if ARCH_X86_32
+ mov r3m, seed
+
+ DEFINE_ARGS luma, src, scaling, offy, w, picptr, offx
+
+ mov offxd, offyd
+%else
+ DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
+ offx, offy, see, left_offxy, unused1, unused2, lstride
+
+ mov offyd, seed
+ mov offxd, seed
+%endif
+ ror offyd, 8
+ shr offxd, 12
+ and offyd, 0xf
+ imul offyd, 82
+ lea offyq, [offyq+offxq+498] ; offy*stride+offx
+
+%if ARCH_X86_32
+ DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut
+%else
+ DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
+ h, offxy, see, left_offxy, unused1, unused2, lstride, luma_bak
+%endif
+
+ mov hd, r7m
+ mov grain_lutq, grain_lutmp
+%%loop_y_h_overlap:
+ ; src
+%if ARCH_X86_32
+ mov lumaq, r9mp
+%endif
+ mova m4, [lumaq+ 0]
+ mova m6, [lumaq+16]
+ mova m0, [srcq]
+%if ARCH_X86_32
+ add lumaq, r10mp
+ mov r9mp, lumaq
+ mov r5, r5m
+ movd m7, [base+pb_1]
+%else
+ movd m7, [pb_1]
+%endif
+ pshufd m7, m7, q0000
+ pxor m2, m2
+ pmaddubsw m4, m7
+ pmaddubsw m6, m7
+ pavgw m4, m2
+ pavgw m6, m2
+
+%if %1
+ packuswb m4, m6 ; luma
+ punpckhbw m6, m4, m0
+ punpcklbw m4, m0 ; { luma, chroma }+ pmaddubsw m6, m14
+ pmaddubsw m4, m14
+ psraw m6, 6
+ psraw m4, 6
+ paddw m6, m15
+ paddw m4, m15
+ packuswb m4, m6 ; pack+unpack = clip
+ punpckhbw m6, m4, m2
+ punpcklbw m4, m2
+%endif
+
+ ; scaling[luma_src]
+%if ARCH_X86_32
+ vpgatherdw m7, m4, scalingq, r0, r5
+ vpgatherdw m5, m6, scalingq, r0, r5
+%else
+ vpgatherdw m7, m4, scalingq, r12, r2
+ vpgatherdw m5, m6, scalingq, r12, r2
+%endif
+ pcmpeqw m1, m1
+ psrlw m1, 8
+ pand m7, m1
+ pand m5, m1
+
+ ; unpack chroma_source
+ punpckhbw m1, m0, m2
+ punpcklbw m0, m2 ; m0-1: src as word
+
+ ; grain = grain_lut[offy+y][offx+x]
+ movu m3, [grain_lutq+offxyq+ 0]
+%if ARCH_X86_32
+ mov r0, [rsp+8*mmsize+0*gprsize]
+ movd m4, [grain_lutq+r0+ 0]
+%else
+ movd m4, [grain_lutq+left_offxyq+ 0]
+%endif
+ punpcklbw m2, m4, m3
+ pmaddubsw m4, m9, m2
+ pmulhrsw m4, m8
+ packsswb m4, m4
+ pand m4, m10
+ pandn m2, m10, m3
+ por m3, m4, m2
+ pxor m4, m4
+ pcmpgtb m4, m3
+ punpcklbw m2, m3, m4
+ punpckhbw m3, m4
+
+ ; noise = round2(scaling[luma_src] * grain, scaling_shift)
+ pmullw m2, m7
+ pmullw m3, m5
+ pmulhrsw m2, m11
+ pmulhrsw m3, m11
+
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
+%endif
+
+ ; dst = clip_pixel(src, noise)
+ paddw m0, m2
+ paddw m1, m3
+ pmaxsw m0, m13
+ pmaxsw m1, m13
+ pminsw m0, m12
+ pminsw m1, m12
+ packuswb m0, m1
+ movifnidn dstq, dstmp
+ mova [dstq+srcq], m0
+
+%if ARCH_X86_32
+ add srcq, r2mp
+ ; lumaq has already been incremented above
+%else
+ add srcq, r10mp
+ lea lumaq, [lumaq+lstrideq*2]
+%endif
+ add grain_lutq, 82
+ dec hw
+ jg %%loop_y_h_overlap
+
+%if ARCH_X86_32
+ DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut
+
+ mov wq, r4m
+%endif
+ add wq, 16
+ jge %%end
+%if ARCH_X86_32
+ mov srcq, r1mp
+ mov lumaq, r11mp
+%else
+ mov srcq, r11mp
+%endif
+ lea lumaq, [luma_bakq+wq*2]
+ add srcq, wq
+%if ARCH_X86_32
+ mov r4m, wq
+ mov r9m, lumaq
+%endif
+
+ ; r8m = sbym
+ test dword r8m, 2
+ jne %%loop_x_hv_overlap
+ jmp %%loop_x_h_overlap
+
+%%end:
+ RET
+
+%%vertical_overlap:
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap
+%else
+ DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, tmp, sby, see, overlap
+%endif
+
+ or overlapd, 2 ; top_overlap: overlap & 2
+ mov r8m, overlapd
+ movzx sbyd, sbyb
+%if ARCH_X86_32
+ imul r4, [fg_dataq+FGData.seed], 0x00010001
+ DEFINE_ARGS tmp, src, scaling, sby, see, picptr, unused
+%else
+ imul seed, [fg_dataq+FGData.seed], 0x00010001
+%endif
+ imul tmpd, sbyd, 173 * 0x00010001
+ imul sbyd, 37 * 0x01000100
+ add tmpd, (105 << 16) | 188
+ add sbyd, (178 << 24) | (141 << 8)
+ and tmpd, 0x00ff00ff
+ and sbyd, 0xff00ff00
+ xor seed, tmpd
+%if ARCH_X86_32
+ xor sbyd, seed ; (cur_seed << 16) | top_seed
+
+ DEFINE_ARGS luma, src, scaling, see, w, picptr, src_bak
+
+ mov r3m, seed
+ mov wq, r4m
+ shl r10mp, 1
+%else
+ xor seed, sbyd ; (cur_seed << 16) | top_seed
+
+ DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
+ tmp, unused2, see, overlap, unused3, src_bak, lstride, luma_bak
+
+ mov lstrideq, r10mp
+%endif
+
+ mov lumaq, r9mp
+ lea src_bakq, [srcq+wq]
+ lea luma_bakq, [lumaq+wq*2]
+ neg wq
+ sub r0mp, srcq
+%if ARCH_X86_32
+ mov r1m, src_bakq
+ mov r11m, luma_bakq
+ mov r4m, wq
+
+ DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2
+%else
+ mov r11mp, src_bakq
+ mov r10mp, strideq
+%endif
+
+%%loop_x_v_overlap:
+%if ARCH_X86_32
+ mov seed, r3m
+ xor tmpd, tmpd
+%endif
+ ; we assume from the block above that bits 8-15 of tmpd are zero'ed
+ mov r6d, seed
+ or seed, 0xeff4eff4
+ test seeb, seeh
+ setp tmpb ; parity of top_seed
+ shr seed, 16
+ shl tmpd, 16
+ test seeb, seeh
+ setp tmpb ; parity of cur_seed
+ or r6d, 0x00010001
+ xor tmpd, r6d
+ mov seed, tmpd
+ ror seed, 1 ; updated (cur_seed << 16) | top_seed
+
+%if ARCH_X86_32
+ mov r3m, seed
+
+ DEFINE_ARGS dst, src, scaling, offy, h, picptr, offx
+
+ mov offxd, offyd
+%else
+ DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
+ offx, offy, see, overlap, top_offxy, unused, lstride
+
+ mov offxd, seed
+ mov offyd, seed
+%endif
+ ror offyd, 8
+ ror offxd, 12
+ and offyd, 0xf000f
+ and offxd, 0xf000f
+ imul offyd, 82
+ ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
+ lea offyq, [offyq+offxq+0x10001*498+16*82]
+
+%if ARCH_X86_32
+ DEFINE_ARGS tmp, src, scaling, offxy, h, picptr, top_offxy
+%else
+ DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
+ h, offxy, see, overlap, top_offxy, unused, lstride, luma_bak
+%endif
+
+ movzx top_offxyd, offxyw
+ shr offxyd, 16
+%if ARCH_X86_32
+ mov [rsp+8*mmsize+1*gprsize], top_offxyd
+
+ DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut
+%endif
+
+ mov hd, r7m
+ mov grain_lutq, grain_lutmp
+%%loop_y_v_overlap:
+%if ARCH_X86_32
+ mov lumaq, r9mp
+%endif
+ mova m4, [lumaq+ 0]
+ mova m6, [lumaq+16]
+ mova m0, [srcq]
+%if ARCH_X86_32
+ add lumaq, r10mp
+ mov r9mp, lumaq
+ mov r5, r5m
+ movd m7, [base+pb_1]
+%else
+ movd m7, [pb_1]
+%endif
+ pshufd m7, m7, q0000
+ pxor m2, m2
+ pmaddubsw m4, m7
+ pmaddubsw m6, m7
+ pavgw m4, m2
+ pavgw m6, m2
+
+%if %1
+ packuswb m4, m6 ; luma
+ punpckhbw m6, m4, m0
+ punpcklbw m4, m0 ; { luma, chroma }+ pmaddubsw m6, m14
+ pmaddubsw m4, m14
+ psraw m6, 6
+ psraw m4, 6
+ paddw m6, m15
+ paddw m4, m15
+ packuswb m4, m6 ; pack+unpack = clip
+ punpckhbw m6, m4, m2
+ punpcklbw m4, m2
+%endif
+
+ ; scaling[luma_src]
+%if ARCH_X86_32
+ vpgatherdw m7, m4, scalingq, r0, r5
+ vpgatherdw m5, m6, scalingq, r0, r5
+%else
+ vpgatherdw m7, m4, scalingq, r12, r2
+ vpgatherdw m5, m6, scalingq, r12, r2
+%endif
+ pcmpeqw m1, m1
+ psrlw m1, 8
+ pand m7, m1
+ pand m5, m1
+
+ ; grain = grain_lut[offy+y][offx+x]
+ movu m3, [grain_lutq+offxyq]
+%if ARCH_X86_32
+ mov r0, [rsp+8*mmsize+1*gprsize]
+ movu m4, [grain_lutq+r0]
+%else
+ movu m4, [grain_lutq+top_offxyq]
+%endif
+ punpckhbw m1, m4, m3
+ punpcklbw m4, m3
+ pmaddubsw m2, m9, m1
+ pmaddubsw m3, m9, m4
+ pmulhrsw m2, m8
+ pmulhrsw m3, m8
+ packsswb m3, m2
+ pxor m1, m1
+ pcmpgtb m1, m3
+ punpcklbw m2, m3, m1
+ punpckhbw m3, m1
+
+ ; noise = round2(scaling[luma_src] * grain, scaling_shift)
+ pmullw m2, m7
+ pmullw m3, m5
+ pmulhrsw m2, m11
+ pmulhrsw m3, m11
+
+ ; unpack chroma_source
+ pxor m4, m4
+ punpckhbw m1, m0, m4
+ punpcklbw m0, m4 ; m0-1: src as word
+
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
+%endif
+
+ ; dst = clip_pixel(src, noise)
+ paddw m0, m2
+ paddw m1, m3
+ pmaxsw m0, m13
+ pmaxsw m1, m13
+ pminsw m0, m12
+ pminsw m1, m12
+ packuswb m0, m1
+ movifnidn dstq, dstmp
+ mova [dstq+srcq], m0
+
+ dec hw
+ je %%end_y_v_overlap
+%if ARCH_X86_32
+ add srcq, r2mp
+ ; lumaq has already been incremented above
+%else
+ add srcq, r10mp
+ lea lumaq, [lumaq+lstrideq*2]
+%endif
+ add grain_lutq, 82
+ jmp %%loop_y
+
+%%end_y_v_overlap:
+%if ARCH_X86_32
+ DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut
+
+ mov wq, r4m
+%endif
+ add wq, 16
+ jge %%end_hv
+%if ARCH_X86_32
+ mov srcq, r1mp
+ mov lumaq, r11mp
+%else
+ mov srcq, r11mp
+%endif
+ lea lumaq, [luma_bakq+wq*2]
+ add srcq, wq
+%if ARCH_X86_32
+ mov r4m, wq
+ mov r9m, lumaq
+%endif
+
+ ; since fg_dataq.overlap is guaranteed to be set, we never jump
+ ; back to .loop_x_v_overlap, and instead always fall-through to
+ ; h+v overlap
+
+%%loop_x_hv_overlap:
+%if ARCH_X86_32
+ DEFINE_ARGS tmp, src, scaling, offxy, w, picptr, unused
+
+ mov r6, [rsp+8*mmsize+1*gprsize]
+ lea r0, [r3d+16]
+ add r6, 16
+ mov [rsp+8*mmsize+0*gprsize], r0 ; left_offxy
+ mov [rsp+8*mmsize+2*gprsize], r6 ; topleft_offxy
+
+ DEFINE_ARGS tmp, src, scaling, see, w, picptr, unused
+
+ mov seed, r3m
+ xor tmpd, tmpd
+%else
+ DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
+ tmp, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride
+
+ lea topleft_offxyq, [top_offxyq+16]
+ lea left_offxyq, [offxyq+16]
+
+ ; we assume from the block above that bits 8-15 of tmpd are zero'ed
+%endif
+ mov r6d, seed
+ or seed, 0xeff4eff4
+ test seeb, seeh
+ setp tmpb ; parity of top_seed
+ shr seed, 16
+ shl tmpd, 16
+ test seeb, seeh
+ setp tmpb ; parity of cur_seed
+ or r6d, 0x00010001
+ xor tmpd, r6d
+ mov seed, tmpd
+ ror seed, 1 ; updated (cur_seed << 16) | top_seed
+
+%if ARCH_X86_32
+ mov r3m, seed
+
+ DEFINE_ARGS tmp, src, scaling, offy, w, picptr, offx
+
+ mov offxd, offyd
+%else
+ DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
+ offx, offy, see, left_offxy, top_offxy, topleft_offxy, lstride
+
+ mov offxd, seed
+ mov offyd, seed
+%endif
+ ror offyd, 8
+ ror offxd, 12
+ and offyd, 0xf000f
+ and offxd, 0xf000f
+ imul offyd, 82
+ ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy
+ lea offyq, [offyq+offxq+0x10001*498+16*82]
+
+%if ARCH_X86_32
+ DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut
+%else
+ DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \
+ h, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride, luma_bak
+%endif
+
+ movzx top_offxyd, offxyw
+ shr offxyd, 16
+%if ARCH_X86_32
+ mov [rsp+8*mmsize+1*gprsize], top_offxyd
+%endif
+
+ mov hd, r7m
+ mov grain_lutq, grain_lutmp
+%%loop_y_hv_overlap:
+ ; src
+%if ARCH_X86_32
+ DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut
+
+ mov lumaq, r9mp
+%endif
+ mova m4, [lumaq+ 0]
+ mova m6, [lumaq+16]
+ mova m0, [srcq]
+%if ARCH_X86_32
+ add lumaq, r10mp
+ mov r9mp, lumaq
+ mov r5, r5m
+ movd m7, [base+pb_1]
+%else
+ movd m7, [pb_1]
+%endif
+ pshufd m7, m7, q0000
+ pxor m2, m2
+ pmaddubsw m4, m7
+ pmaddubsw m6, m7
+ pavgw m4, m2
+ pavgw m6, m2
+
+%if %1
+ packuswb m4, m6 ; luma
+ punpckhbw m6, m4, m0
+ punpcklbw m4, m0 ; { luma, chroma }+ pmaddubsw m6, m14
+ pmaddubsw m4, m14
+ psraw m6, 6
+ psraw m4, 6
+ paddw m6, m15
+ paddw m4, m15
+ packuswb m4, m6 ; pack+unpack = clip
+ punpckhbw m6, m4, m2
+ punpcklbw m4, m2
+%endif
+
+ ; scaling[src]
+%if ARCH_X86_32
+ vpgatherdw m7, m4, scalingq, r0, r5
+ vpgatherdw m5, m6, scalingq, r0, r5
+%else
+ movd m1, [grain_lutq+topleft_offxyq]
+ vpgatherdw m7, m4, scalingq, r2, r12
+ vpgatherdw m5, m6, scalingq, r2, r12
+%endif
+ pcmpeqw m2, m2
+ psrlw m2, 8
+ pand m7, m2
+ pand m5, m2
+
+ ; grain = grain_lut[offy+y][offx+x]
+%if ARCH_X86_32
+ mov r0, [rsp+8*mmsize+2*gprsize] ; topleft_offxy
+ mov r5, [rsp+8*mmsize+1*gprsize] ; top_offxy
+ movd m1, [grain_lutq+r0]
+ mov r0, [rsp+8*mmsize+0*gprsize] ; left_offxy
+%endif
+ movu m3, [grain_lutq+offxyq]
+%if ARCH_X86_32
+ movu m6, [grain_lutq+r5]
+ movd m4, [grain_lutq+r0]
+%else
+ movu m6, [grain_lutq+top_offxyq]
+ movd m4, [grain_lutq+left_offxyq]
+%endif
+ ; do h interpolation first (so top | top/left -> top, left | cur -> cur)
+ punpcklbw m1, m6
+ punpcklbw m4, m3
+ punpcklwd m4, m1
+ pmaddubsw m1, m9, m4
+ pmulhrsw m1, m8
+ packsswb m1, m1
+ pandn m4, m10, m3
+ pandn m3, m10, m6
+ psrldq m6, m1, 1
+ pand m1, m10
+ pand m6, m10
+ por m4, m1
+ por m3, m6
+ ; followed by v interpolation (top | cur -> cur)
+ punpckhbw m1, m3, m4
+ punpcklbw m3, m4
+ pmaddubsw m4, m9, m1
+ pmaddubsw m1, m9, m3
+ pmulhrsw m4, m8
+ pmulhrsw m1, m8
+ packsswb m1, m4
+ pxor m4, m4
+ pcmpgtb m4, m1
+ punpcklbw m2, m1, m4
+ punpckhbw m1, m4
+
+ ; noise = round2(scaling[src] * grain, scaling_shift)
+ pmullw m2, m7
+ pmullw m1, m5
+ pmulhrsw m2, m11
+ pmulhrsw m1, m11
+
+%if ARCH_X86_32
+ DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut
+%endif
+
+ ; unpack chroma source
+ pxor m4, m4
+ punpckhbw m3, m0, m4
+ punpcklbw m0, m4 ; m0-1: src as word
+
+ ; dst = clip_pixel(src, noise)
+ paddw m0, m2
+ paddw m3, m1
+ pmaxsw m0, m13
+ pmaxsw m3, m13
+ pminsw m0, m12
+ pminsw m3, m12
+ packuswb m0, m3
+ movifnidn dstq, dstmp
+ mova [dstq+srcq], m0
+
+%if ARCH_X86_32
+ add srcq, r2mp
+ ; lumaq has been adjusted above already
+%else
+ add srcq, r10mp
+ lea lumaq, [lumaq+lstrideq*2]
+%endif
+ add grain_lutq, 82
+ dec hw
+ jg %%loop_y_h_overlap
+
+%if ARCH_X86_32
+ DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut
+
+ mov wq, r4m
+%endif
+ add wq, 16
+ jge %%end_hv
+%if ARCH_X86_32
+ mov srcq, r1mp
+ mov lumaq, r11mp
+%else
+ mov srcq, r11mp
+%endif
+ lea lumaq, [luma_bakq+wq*2]
+ add srcq, wq
+%if ARCH_X86_32
+ mov r4m, wq
+ mov r9m, lumaq
+%endif
+ jmp %%loop_x_hv_overlap
+
+%%end_hv:
+ RET
+%endmacro
+
+ FGUV_32x32xN_LOOP 1
+.csfl:
+ FGUV_32x32xN_LOOP 0
--- a/tests/checkasm/filmgrain.c
+++ b/tests/checkasm/filmgrain.c
@@ -49,22 +49,22 @@
for (int i = 0; i < 4; i++) { if (check_func(dsp->generate_grain_y, "gen_grain_y_ar%d_%dbpc", i, BITDEPTH)) {- Dav1dFilmGrainData fg_data;
- fg_data.seed = rnd() & 0xFFFF;
+ ALIGN_STK_16(Dav1dFilmGrainData, fg_data, 1,);
+ fg_data[0].seed = rnd() & 0xFFFF;
#if BITDEPTH == 16
const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff;
#endif
- fg_data.grain_scale_shift = rnd() & 3;
- fg_data.ar_coeff_shift = (rnd() & 3) + 6;
- fg_data.ar_coeff_lag = i;
- const int num_y_pos = 2 * fg_data.ar_coeff_lag * (fg_data.ar_coeff_lag + 1);
+ fg_data[0].grain_scale_shift = rnd() & 3;
+ fg_data[0].ar_coeff_shift = (rnd() & 3) + 6;
+ fg_data[0].ar_coeff_lag = i;
+ const int num_y_pos = 2 * fg_data[0].ar_coeff_lag * (fg_data[0].ar_coeff_lag + 1);
for (int n = 0; n < num_y_pos; n++)
- fg_data.ar_coeffs_y[n] = (rnd() & 0xff) - 128;
+ fg_data[0].ar_coeffs_y[n] = (rnd() & 0xff) - 128;
- call_ref(grain_lut_c, &fg_data HIGHBD_TAIL_SUFFIX);
- call_new(grain_lut_a, &fg_data HIGHBD_TAIL_SUFFIX);
+ call_ref(grain_lut_c, fg_data HIGHBD_TAIL_SUFFIX);
+ call_new(grain_lut_a, fg_data HIGHBD_TAIL_SUFFIX);
if (memcmp(grain_lut_c, grain_lut_a,
GRAIN_WIDTH * GRAIN_HEIGHT * sizeof(entry)))
{@@ -71,7 +71,7 @@
fail();
}
- bench_new(grain_lut_a, &fg_data HIGHBD_TAIL_SUFFIX);
+ bench_new(grain_lut_a, fg_data HIGHBD_TAIL_SUFFIX);
}
}
@@ -97,38 +97,38 @@
"gen_grain_uv_ar%d_%dbpc_%s",
i, BITDEPTH, ss_name[layout_idx]))
{- Dav1dFilmGrainData fg_data;
- fg_data.seed = rnd() & 0xFFFF;
+ ALIGN_STK_16(Dav1dFilmGrainData, fg_data, 1,);
+ fg_data[0].seed = rnd() & 0xFFFF;
#if BITDEPTH == 16
const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff;
#endif
- fg_data.num_y_points = rnd() & 1;
- fg_data.grain_scale_shift = rnd() & 3;
- fg_data.ar_coeff_shift = (rnd() & 3) + 6;
- fg_data.ar_coeff_lag = i;
- const int num_y_pos = 2 * fg_data.ar_coeff_lag * (fg_data.ar_coeff_lag + 1);
+ fg_data[0].num_y_points = rnd() & 1;
+ fg_data[0].grain_scale_shift = rnd() & 3;
+ fg_data[0].ar_coeff_shift = (rnd() & 3) + 6;
+ fg_data[0].ar_coeff_lag = i;
+ const int num_y_pos = 2 * fg_data[0].ar_coeff_lag * (fg_data[0].ar_coeff_lag + 1);
for (int n = 0; n < num_y_pos; n++)
- fg_data.ar_coeffs_y[n] = (rnd() & 0xff) - 128;
- dsp->generate_grain_y(grain_lut_y, &fg_data HIGHBD_TAIL_SUFFIX);
+ fg_data[0].ar_coeffs_y[n] = (rnd() & 0xff) - 128;
+ dsp->generate_grain_y(grain_lut_y, fg_data HIGHBD_TAIL_SUFFIX);
const int uv = rnd() & 1;
- const int num_uv_pos = num_y_pos + !!fg_data.num_y_points;
+ const int num_uv_pos = num_y_pos + !!fg_data[0].num_y_points;
for (int n = 0; n < num_uv_pos; n++)
- fg_data.ar_coeffs_uv[uv][n] = (rnd() & 0xff) - 128;
- if (!fg_data.num_y_points)
- fg_data.ar_coeffs_uv[uv][num_uv_pos] = 0;
+ fg_data[0].ar_coeffs_uv[uv][n] = (rnd() & 0xff) - 128;
+ if (!fg_data[0].num_y_points)
+ fg_data[0].ar_coeffs_uv[uv][num_uv_pos] = 0;
memset(grain_lut_c, 0xff, sizeof(grain_lut_c));
memset(grain_lut_a, 0xff, sizeof(grain_lut_a));
- call_ref(grain_lut_c, grain_lut_y, &fg_data, uv HIGHBD_TAIL_SUFFIX);
- call_new(grain_lut_a, grain_lut_y, &fg_data, uv HIGHBD_TAIL_SUFFIX);
+ call_ref(grain_lut_c, grain_lut_y, fg_data, uv HIGHBD_TAIL_SUFFIX);
+ call_new(grain_lut_a, grain_lut_y, fg_data, uv HIGHBD_TAIL_SUFFIX);
int diff = 0, w = ss_x ? 44 : GRAIN_WIDTH;
for (int y = 0; y < (ss_y ? 38 : GRAIN_HEIGHT); y++)
diff |= memcmp(grain_lut_a[y], grain_lut_c[y], w * sizeof(entry));
if (diff) fail();
- bench_new(grain_lut_a, grain_lut_y, &fg_data, uv HIGHBD_TAIL_SUFFIX);
+ bench_new(grain_lut_a, grain_lut_y, fg_data, uv HIGHBD_TAIL_SUFFIX);
}
}
}
@@ -149,8 +149,8 @@
int bh, int row_num HIGHBD_DECL_SUFFIX);
if (check_func(dsp->fgy_32x32xn, "fgy_32x32xn_%dbpc", BITDEPTH)) {- Dav1dFilmGrainData fg_data;
- fg_data.seed = rnd() & 0xFFFF;
+ ALIGN_STK_16(Dav1dFilmGrainData, fg_data, 16,);
+ fg_data[0].seed = rnd() & 0xFFFF;
#if BITDEPTH == 16
const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff;
@@ -160,23 +160,23 @@
uint8_t scaling[SCALING_SIZE];
entry grain_lut[GRAIN_HEIGHT + 1][GRAIN_WIDTH];
- fg_data.grain_scale_shift = rnd() & 3;
- fg_data.ar_coeff_shift = (rnd() & 3) + 6;
- fg_data.ar_coeff_lag = rnd() & 3;
- const int num_y_pos = 2 * fg_data.ar_coeff_lag * (fg_data.ar_coeff_lag + 1);
+ fg_data[0].grain_scale_shift = rnd() & 3;
+ fg_data[0].ar_coeff_shift = (rnd() & 3) + 6;
+ fg_data[0].ar_coeff_lag = rnd() & 3;
+ const int num_y_pos = 2 * fg_data[0].ar_coeff_lag * (fg_data[0].ar_coeff_lag + 1);
for (int n = 0; n < num_y_pos; n++)
- fg_data.ar_coeffs_y[n] = (rnd() & 0xff) - 128;
- dsp->generate_grain_y(grain_lut, &fg_data HIGHBD_TAIL_SUFFIX);
+ fg_data[0].ar_coeffs_y[n] = (rnd() & 0xff) - 128;
+ dsp->generate_grain_y(grain_lut, fg_data HIGHBD_TAIL_SUFFIX);
- fg_data.num_y_points = 2 + (rnd() % 13);
- const int pad = 0xff / fg_data.num_y_points;
- for (int n = 0; n < fg_data.num_y_points; n++) {- fg_data.y_points[n][0] = 0xff * n / fg_data.num_y_points;
- fg_data.y_points[n][0] += rnd() % pad;
- fg_data.y_points[n][1] = rnd() & 0xff;
+ fg_data[0].num_y_points = 2 + (rnd() % 13);
+ const int pad = 0xff / fg_data[0].num_y_points;
+ for (int n = 0; n < fg_data[0].num_y_points; n++) {+ fg_data[0].y_points[n][0] = 0xff * n / fg_data[0].num_y_points;
+ fg_data[0].y_points[n][0] += rnd() % pad;
+ fg_data[0].y_points[n][1] = rnd() & 0xff;
}
- generate_scaling(bitdepth_from_max(bitdepth_max), fg_data.y_points,
- fg_data.num_y_points, scaling);
+ generate_scaling(bitdepth_from_max(bitdepth_max), fg_data[0].y_points,
+ fg_data[0].num_y_points, scaling);
const int w = 1 + (rnd() & 127);
const int h = 1 + (rnd() & 31);
@@ -186,20 +186,20 @@
src[y * PXSTRIDE(stride) + x] = rnd() & bitdepth_max;
const int row_num = rnd() & 1 ? rnd() & 0x7ff : 0;
- fg_data.clip_to_restricted_range = rnd() & 1;
- fg_data.scaling_shift = (rnd() & 3) + 8;
- for (fg_data.overlap_flag = 0; fg_data.overlap_flag <= 1;
- fg_data.overlap_flag++)
+ fg_data[0].clip_to_restricted_range = rnd() & 1;
+ fg_data[0].scaling_shift = (rnd() & 3) + 8;
+ for (fg_data[0].overlap_flag = 0; fg_data[0].overlap_flag <= 1;
+ fg_data[0].overlap_flag++)
{- call_ref(c_dst, src, stride, &fg_data, w, scaling, grain_lut, h,
+ call_ref(c_dst, src, stride, fg_data, w, scaling, grain_lut, h,
row_num HIGHBD_TAIL_SUFFIX);
- call_new(a_dst, src, stride, &fg_data, w, scaling, grain_lut, h,
+ call_new(a_dst, src, stride, fg_data, w, scaling, grain_lut, h,
row_num HIGHBD_TAIL_SUFFIX);
checkasm_check_pixel(c_dst, stride, a_dst, stride, w, h, "dst");
}
- fg_data.overlap_flag = 1;
- bench_new(a_dst, src, stride, &fg_data, 64, scaling, grain_lut, 32,
+ fg_data[0].overlap_flag = 1;
+ bench_new(a_dst, src, stride, fg_data, 64, scaling, grain_lut, 32,
row_num HIGHBD_TAIL_SUFFIX);
}
@@ -231,9 +231,9 @@
"fguv_32x32xn_%dbpc_%s_csfl%d",
BITDEPTH, ss_name[layout_idx], csfl))
{- Dav1dFilmGrainData fg_data;
+ ALIGN_STK_16(Dav1dFilmGrainData, fg_data, 1,);
- fg_data.seed = rnd() & 0xFFFF;
+ fg_data[0].seed = rnd() & 0xFFFF;
#if BITDEPTH == 16
const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff;
@@ -245,15 +245,18 @@
uint8_t scaling[SCALING_SIZE];
entry grain_lut[2][GRAIN_HEIGHT + 1][GRAIN_WIDTH];
- fg_data.grain_scale_shift = rnd() & 3;
- fg_data.ar_coeff_shift = (rnd() & 3) + 6;
- fg_data.ar_coeff_lag = rnd() & 3;
- const int num_y_pos = 2 * fg_data.ar_coeff_lag * (fg_data.ar_coeff_lag + 1);
+ fg_data[0].grain_scale_shift = rnd() & 3;
+ fg_data[0].ar_coeff_shift = (rnd() & 3) + 6;
+ fg_data[0].ar_coeff_lag = rnd() & 3;
+ const int num_y_pos = 2 * fg_data[0].ar_coeff_lag * (fg_data[0].ar_coeff_lag + 1);
for (int n = 0; n < num_y_pos; n++)
- fg_data.ar_coeffs_y[n] = (rnd() & 0xff) - 128;
- dsp->generate_grain_y(grain_lut[0], &fg_data HIGHBD_TAIL_SUFFIX);
+ fg_data[0].ar_coeffs_y[n] = (rnd() & 0xff) - 128;
+ const int num_uv_pos = num_y_pos + 1;
+ for (int n = 0; n < num_uv_pos; n++)
+ fg_data[0].ar_coeffs_uv[uv_pl][n] = (rnd() & 0xff) - 128;
+ dsp->generate_grain_y(grain_lut[0], fg_data HIGHBD_TAIL_SUFFIX);
dsp->generate_grain_uv[layout_idx](grain_lut[1], grain_lut[0],
- &fg_data, uv_pl HIGHBD_TAIL_SUFFIX);
+ fg_data, uv_pl HIGHBD_TAIL_SUFFIX);
const int w = 1 + (rnd() & (127 >> ss_x));
const int h = 1 + (rnd() & (31 >> ss_y));
@@ -268,47 +271,47 @@
const int row_num = rnd() & 1 ? rnd() & 0x7ff : 0;
if (csfl) {- fg_data.num_y_points = 2 + (rnd() % 13);
- const int pad = 0xff / fg_data.num_y_points;
- for (int n = 0; n < fg_data.num_y_points; n++) {- fg_data.y_points[n][0] = 0xff * n / fg_data.num_y_points;
- fg_data.y_points[n][0] += rnd() % pad;
- fg_data.y_points[n][1] = rnd() & 0xff;
+ fg_data[0].num_y_points = 2 + (rnd() % 13);
+ const int pad = 0xff / fg_data[0].num_y_points;
+ for (int n = 0; n < fg_data[0].num_y_points; n++) {+ fg_data[0].y_points[n][0] = 0xff * n / fg_data[0].num_y_points;
+ fg_data[0].y_points[n][0] += rnd() % pad;
+ fg_data[0].y_points[n][1] = rnd() & 0xff;
}
- generate_scaling(bitdepth_from_max(bitdepth_max), fg_data.y_points,
- fg_data.num_y_points, scaling);
+ generate_scaling(bitdepth_from_max(bitdepth_max), fg_data[0].y_points,
+ fg_data[0].num_y_points, scaling);
} else {- fg_data.num_uv_points[uv_pl] = 2 + (rnd() % 9);
- const int pad = 0xff / fg_data.num_uv_points[uv_pl];
- for (int n = 0; n < fg_data.num_uv_points[uv_pl]; n++) {- fg_data.uv_points[uv_pl][n][0] = 0xff * n / fg_data.num_uv_points[uv_pl];
- fg_data.uv_points[uv_pl][n][0] += rnd() % pad;
- fg_data.uv_points[uv_pl][n][1] = rnd() & 0xff;
+ fg_data[0].num_uv_points[uv_pl] = 2 + (rnd() % 9);
+ const int pad = 0xff / fg_data[0].num_uv_points[uv_pl];
+ for (int n = 0; n < fg_data[0].num_uv_points[uv_pl]; n++) {+ fg_data[0].uv_points[uv_pl][n][0] = 0xff * n / fg_data[0].num_uv_points[uv_pl];
+ fg_data[0].uv_points[uv_pl][n][0] += rnd() % pad;
+ fg_data[0].uv_points[uv_pl][n][1] = rnd() & 0xff;
}
- generate_scaling(bitdepth_from_max(bitdepth_max), fg_data.uv_points[uv_pl],
- fg_data.num_uv_points[uv_pl], scaling);
+ generate_scaling(bitdepth_from_max(bitdepth_max), fg_data[0].uv_points[uv_pl],
+ fg_data[0].num_uv_points[uv_pl], scaling);
- fg_data.uv_mult[uv_pl] = (rnd() & 0xff) - 128;
- fg_data.uv_luma_mult[uv_pl] = (rnd() & 0xff) - 128;
- fg_data.uv_offset[uv_pl] = (rnd() & 0x1ff) - 256;
+ fg_data[0].uv_mult[uv_pl] = (rnd() & 0xff) - 128;
+ fg_data[0].uv_luma_mult[uv_pl] = (rnd() & 0xff) - 128;
+ fg_data[0].uv_offset[uv_pl] = (rnd() & 0x1ff) - 256;
}
- fg_data.clip_to_restricted_range = rnd() & 1;
- fg_data.scaling_shift = (rnd() & 3) + 8;
- fg_data.chroma_scaling_from_luma = csfl;
- for (fg_data.overlap_flag = 0; fg_data.overlap_flag <= 1;
- fg_data.overlap_flag++)
+ fg_data[0].clip_to_restricted_range = rnd() & 1;
+ fg_data[0].scaling_shift = (rnd() & 3) + 8;
+ fg_data[0].chroma_scaling_from_luma = csfl;
+ for (fg_data[0].overlap_flag = 0; fg_data[0].overlap_flag <= 1;
+ fg_data[0].overlap_flag++)
{- call_ref(c_dst, src, stride, &fg_data, w, scaling, grain_lut[1], h,
+ call_ref(c_dst, src, stride, fg_data, w, scaling, grain_lut[1], h,
row_num, luma_src, lstride, uv_pl, is_identity HIGHBD_TAIL_SUFFIX);
- call_new(a_dst, src, stride, &fg_data, w, scaling, grain_lut[1], h,
+ call_new(a_dst, src, stride, fg_data, w, scaling, grain_lut[1], h,
row_num, luma_src, lstride, uv_pl, is_identity HIGHBD_TAIL_SUFFIX);
checkasm_check_pixel(c_dst, stride, a_dst, stride, w, h, "dst");
}
- fg_data.overlap_flag = 1;
- bench_new(a_dst, src, stride, &fg_data, 32, scaling, grain_lut[1], 16,
+ fg_data[0].overlap_flag = 1;
+ bench_new(a_dst, src, stride, fg_data, 32, scaling, grain_lut[1], 16,
row_num, luma_src, lstride, uv_pl, is_identity HIGHBD_TAIL_SUFFIX);
}
}