| ;****************************************************************************** |
| ;* VP9 IDCT SIMD optimizations |
| ;* |
| ;* Copyright (C) 2025 Two Orioles, LLC |
| ;* |
| ;* This file is part of FFmpeg. |
| ;* |
| ;* FFmpeg is free software; you can redistribute it and/or |
| ;* modify it under the terms of the GNU Lesser General Public |
| ;* License as published by the Free Software Foundation; either |
| ;* version 2.1 of the License, or (at your option) any later version. |
| ;* |
| ;* FFmpeg is distributed in the hope that it will be useful, |
| ;* but WITHOUT ANY WARRANTY; without even the implied warranty of |
| ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| ;* Lesser General Public License for more details. |
| ;* |
| ;* You should have received a copy of the GNU Lesser General Public |
| ;* License along with FFmpeg; if not, write to the Free Software |
| ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| ;****************************************************************************** |
| |
| %include "libavutil/x86/x86util.asm" |
| |
| %if ARCH_X86_64 && HAVE_AVX512ICL_EXTERNAL |
| |
| SECTION_RODATA 64 |
| |
| ; The following set of constants are ordered to form the |
| ; qword shuffle mask { 0, 2, 4, 6, 1, 3, 5, 7 } |
| %define deintq_perm pd_5520 |
| pd_5520: dd 5520 |
| pd_9760: dd 9760 |
| pd_10394: dd 10394 |
| pd_15426: dd 15426 |
| pd_804: dd 804 |
| pd_2404: dd 2404 |
| pd_6270: dd 6270 |
| pd_9102: dd 9102 |
| pd_11585: dd 11585 |
| pd_12665: dd 12665 |
| pd_7723: dd 7723 |
| pd_14811: dd 14811 |
| pd_7005: dd 7005 |
| pd_14053: dd 14053 |
| pd_8423: dd 8423 |
| pd_13623: dd 13623 |
| |
| pixel_clip: times 2 dw 0x7c00 |
| pixel_clip6: dd 2031648 ; 32 + (pixel_clip << 6) |
| pd_532480: dd 532480 ; 8192 + (32 << 14) |
| pd_8192: dd 8192 |
| |
| pd_1606: dd 1606 |
| pd_3196: dd 3196 |
| pd_3981: dd 3981 |
| pd_4756: dd 4756 |
| pd_11003: dd 11003 |
| pd_12140: dd 12140 |
| pd_13160: dd 13160 |
| pd_14449: dd 14449 |
| pd_15137: dd 15137 |
| pd_15679: dd 15679 |
| pd_15893: dd 15893 |
| pd_16069: dd 16069 |
| pd_16207: dd 16207 |
| pd_16305: dd 16305 |
| pd_16364: dd 16364 |
| |
| SECTION .text |
| |
| %define o_base (deintq_perm+128) |
| %define o(x) (r5 - o_base + (x)) |
| %define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) |
| |
| ; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 |
| ; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 |
| ; skip round/shift if rnd is not a number |
| %macro ITX_MULSUB_2D 8-9 0 ; dst/src[1-2], tmp[1-3], rnd, coef[1-2], inv_dst2 |
| %if %8 < 32 |
| pmulld m%4, m%1, m%8 |
| pmulld m%3, m%2, m%8 |
| %else |
| vpbroadcastd m%3, [o(pd_%8)] |
| pmulld m%4, m%1, m%3 |
| pmulld m%3, m%2 |
| %endif |
| %if %7 < 32 |
| pmulld m%1, m%7 |
| pmulld m%2, m%7 |
| %else |
| vpbroadcastd m%5, [o(pd_%7)] |
| pmulld m%1, m%5 |
| pmulld m%2, m%5 |
| %endif |
| %if %9 |
| psubd m%4, m%6, m%4 |
| psubd m%2, m%4, m%2 |
| %else |
| %ifnum %6 |
| paddd m%4, m%6 |
| %endif |
| paddd m%2, m%4 |
| %endif |
| %ifnum %6 |
| paddd m%1, m%6 |
| %endif |
| psubd m%1, m%3 |
| %ifnum %6 |
| psrad m%2, 14 |
| psrad m%1, 14 |
| %endif |
| %endmacro |
| |
| %macro WRAP_YMM 1+ |
| INIT_YMM cpuname |
| %1 |
| INIT_ZMM cpuname |
| %endmacro |
| |
| %macro TRANSPOSE_4D 5 ; in[1-4], tmp |
| punpckhdq m%5, m%3, m%4 ; c2 d2 c3 d3 |
| punpckldq m%3, m%4 ; c0 d0 c1 d1 |
| punpckhdq m%4, m%1, m%2 ; a2 b2 a3 b3 |
| punpckldq m%1, m%2 ; a0 b0 a1 b1 |
| punpckhqdq m%2, m%1, m%3 ; a1 b1 c1 d1 |
| punpcklqdq m%1, m%3 ; a0 b0 c0 d0 |
| punpcklqdq m%3, m%4, m%5 ; a2 b2 c2 d2 |
| punpckhqdq m%4, m%5 ; a3 b3 c3 d3 |
| %endmacro |
| |
| %macro TRANSPOSE_4DQ 5 ; in[1-4], tmp |
| vshufi32x4 m%5, m%3, m%4, q3232 ; c2 c3 d2 d3 |
| vinserti32x8 m%3, ym%4, 1 ; c0 c1 d0 d1 |
| vshufi32x4 m%4, m%1, m%2, q3232 ; a2 a3 b2 b3 |
| vinserti32x8 m%1, ym%2, 1 ; a0 a1 b0 b1 |
| vshufi32x4 m%2, m%1, m%3, q3131 ; a1 b1 c1 d1 |
| vshufi32x4 m%1, m%3, q2020 ; a0 b0 c0 d0 |
| vshufi32x4 m%3, m%4, m%5, q2020 ; a2 b2 c2 d2 |
| vshufi32x4 m%4, m%5, q3131 ; a3 b3 c3 d3 |
| %endmacro |
| |
| %macro INV_TXFM_FN 3-4 0 ; type1, type2, size, eob_offset |
| cglobal vp9_i%1_i%2_%3_add_10, 4, 5, 0, dst, stride, c, eob, tx2 |
| %define %%p1 m(vp9_i%1_%3_internal_10) |
| lea r5, [o_base] |
| ; Jump to the 1st txfm function if we're not taking the fast path, which |
| ; in turn performs an indirect jump to the 2nd txfm function. |
| lea tx2q, [m(vp9_i%2_%3_internal_10).pass2] |
| %ifidn %1_%2, dct_dct |
| dec eobd |
| jnz %%p1 |
| %else |
| %if %4 |
| add eobd, %4 |
| %endif |
| ; jump to the 1st txfm function unless it's located directly after this |
| times ((%%end - %%p1) >> 31) & 1 jmp %%p1 |
| ALIGN function_align |
| %%end: |
| %endif |
| %endmacro |
| |
| %macro INV_TXFM_16X16_FN 2-3 0 ; type1, type2, eob_offset |
| INV_TXFM_FN %1, %2, 16x16, %3 |
| %ifidn %1_%2, dct_dct |
| imul r6d, [cq], 11585 |
| vpbroadcastd ym3, [o(pixel_clip)] |
| mov [cq], r3d |
| add r6d, 8192 |
| sar r6d, 14 |
| imul r6d, 11585 |
| or r3d, 8 |
| add r6d, 532480 |
| sar r6d, 20 |
| vpbroadcastw ym2, r6d |
| paddsw ym2, ym3 |
| .dconly_loop: |
| paddsw ym0, ym2, [dstq+strideq*0] |
| paddsw ym1, ym2, [dstq+strideq*1] |
| psubusw ym0, ym3 |
| psubusw ym1, ym3 |
| mova [dstq+strideq*0], ym0 |
| mova [dstq+strideq*1], ym1 |
| lea dstq, [dstq+strideq*2] |
| dec r3d |
| jg .dconly_loop |
| RET |
| %endif |
| %endmacro |
| |
| %macro IDCT16_PART1 0 |
| %if mmsize == 64 |
| .main_part1_fast: |
| %endif |
| pmulld m15, m1, [o(pd_16305)] {bcstd} ; t15a |
| pmulld m1, [o(pd_1606)] {bcstd} ; t8a |
| pmulld m9, m7, [o(pd_10394)] {bcstd} ; t9a |
| pmulld m7, [o(pd_12665)] {bcstd} ; t14a |
| pmulld m11, m5, [o(pd_14449)] {bcstd} ; t13a |
| pmulld m5, [o(pd_7723)] {bcstd} ; t10a |
| pmulld m13, m3, [o(pd_4756)] {bcstd} ; t11a |
| pmulld m3, [o(pd_15679)] {bcstd} ; t12a |
| pmulld m10, m6, [o(pd_9102)] {bcstd} ; t5a |
| pmulld m6, [o(pd_13623)] {bcstd} ; t6a |
| pmulld m14, m2, [o(pd_16069)] {bcstd} ; t7a |
| pmulld m2, [o(pd_3196)] {bcstd} ; t4a |
| pmulld m12, m4, [o(pd_15137)] {bcstd} ; t3 |
| pmulld m4, [o(pd_6270)] {bcstd} ; t2 |
| pmulld m0, m21 |
| REPX {psubd x, m20, x}, m9, m13, m10 |
| paddd m0, m20 |
| mova m18, m0 |
| %if mmsize == 64 ; for the ymm variant we only ever use the fast path |
| jmp %%main_part1b |
| .main_part1: |
| ITX_MULSUB_2D 1, 15, 16, 17, 18, _, 1606, 16305 ; t8a, t15a |
| ITX_MULSUB_2D 9, 7, 16, 17, 18, _, 12665, 10394 ; t9a, t14a |
| ITX_MULSUB_2D 5, 11, 16, 17, 18, _, 7723, 14449 ; t10a, t13a |
| ITX_MULSUB_2D 13, 3, 16, 17, 18, _, 15679, 4756 ; t11a, t12a |
| ITX_MULSUB_2D 10, 6, 16, 17, 18, _, 13623, 9102 ; t5a, t6a |
| ITX_MULSUB_2D 2, 14, 16, 17, 18, _, 3196, 16069 ; t4a, t7a |
| ITX_MULSUB_2D 4, 12, 16, 17, 18, _, 6270, 15137 ; t2, t3 |
| pmulld m0, m21 |
| pmulld m8, m21 |
| REPX {paddd x, m20}, m0, m9, m13, m10 |
| psubd m18, m0, m8 ; t1 |
| paddd m0, m8 ; t0 |
| %%main_part1b: |
| %endif |
| vpbroadcastd m19, [o(pd_15137)] |
| vpbroadcastd m16, [o(pd_6270)] |
| REPX {paddd x, m20}, m15, m7, m1, m11, m3, m5 |
| REPX {psrad x, 14 }, m15, m7, m1, m9, m11, m3, m5, m13 |
| paddd m17, m15, m7 ; t15 |
| psubd m15, m7 ; t14 |
| psubd m7, m3, m11 ; t13 |
| paddd m3, m11 ; t12 |
| psubd m11, m13, m5 ; t10 |
| paddd m5, m13 ; t11 |
| psubd m13, m1, m9 ; t9 |
| paddd m1, m9 ; t8 |
| ITX_MULSUB_2D 15, 13, 8, 9, _, 20, 16, 19 ; t9a, t14a |
| ITX_MULSUB_2D 7, 11, 8, 9, _, 20, 16, 19, 2 ; t13a, t10a |
| paddd m16, m1, m5 ; t8a |
| psubd m1, m5 ; t11a |
| paddd m8, m15, m11 ; t9 |
| psubd m15, m11 ; t10 |
| psubd m11, m17, m3 ; t12a |
| paddd m17, m3 ; t15a |
| psubd m9, m13, m7 ; t13 |
| paddd m13, m7 ; t14 |
| REPX {pmulld x, m21}, m11, m9, m1, m15 |
| REPX {paddd x, m20}, m2, m6, m14 |
| REPX {psrad x, 14 }, m10, m2, m6, m14 |
| psubd m3, m2, m10 ; t5a |
| paddd m10, m2 ; t4 |
| paddd m11, m20 |
| psubd m5, m11, m1 ; t11 |
| paddd m11, m1 ; t12 |
| psubd m1, m14, m6 ; t6a |
| paddd m14, m6 ; t7 |
| pmulld m1, m21 |
| pmulld m3, m21 |
| paddd m4, m20 |
| paddd m12, m20 |
| REPX {psrad x, 14 }, m4, m12, m0, m18 |
| paddd m9, m20 |
| paddd m2, m9, m15 ; t13a |
| psubd m9, m15 ; t10a |
| paddd m1, m20 |
| psubd m6, m1, m3 ; t5 |
| paddd m1, m3 ; t6 |
| REPX {psrad x, 14}, m6, m1, m11, m5, m2, m9 |
| %endmacro |
| |
| %macro IDCT16_PART2 0 |
| psubd m3, m0, m12 ; t3 |
| paddd m0, m12 ; t0 |
| psubd m12, m18, m4 ; t2 |
| paddd m18, m4 ; t1 |
| psubd m4, m3, m10 ; t4 |
| paddd m3, m10 ; t3 |
| psubd m10, m12, m6 ; t5 |
| paddd m12, m6 ; t2 |
| psubd m6, m18, m1 ; t6 |
| paddd m1, m18 ; t1 |
| psubd m7, m0, m14 ; t7 |
| paddd m0, m14 ; t0 |
| psubd m15, m0, m17 ; out15 |
| paddd m0, m17 ; out0 |
| psubd m14, m1, m13 ; out14 |
| paddd m1, m13 ; out1 |
| psubd m13, m12, m2 ; out13 |
| paddd m2, m12 ; out2 |
| psubd m12, m3, m11 ; out12 |
| paddd m3, m11 ; out3 |
| psubd m11, m4, m5 ; out11 |
| paddd m4, m5 ; out4 |
| paddd m5, m10, m9 ; out5 |
| psubd m10, m9 ; out10 |
| psubd m9, m6, m8 ; out9 |
| paddd m6, m8 ; out6 |
| psubd m8, m7, m16 ; out8 |
| paddd m7, m16 ; out7 |
| %endmacro |
| |
| INIT_ZMM avx512icl |
| INV_TXFM_16X16_FN dct, dct |
| INV_TXFM_16X16_FN dct, adst, 39-23-1 |
| |
| cglobal vp9_idct_16x16_internal_10, 0, 7, 22, dst, stride, c, eob, tx2 |
| mova m0, [cq+64* 0] |
| mova m1, [cq+64* 1] |
| mova m2, [cq+64* 2] |
| mova m3, [cq+64* 3] |
| mova m4, [cq+64* 4] |
| mova m5, [cq+64* 5] |
| mova m6, [cq+64* 6] |
| mova m7, [cq+64* 7] |
| vpbroadcastd m20, [o(pd_8192)] |
| vpbroadcastd m21, [o(pd_11585)] |
| sub eobd, 38 |
| jl .pass1_fast |
| mova m8, [cq+64* 8] |
| mova m9, [cq+64* 9] |
| mova m10, [cq+64*10] |
| mova m11, [cq+64*11] |
| mova m12, [cq+64*12] |
| mova m13, [cq+64*13] |
| mova m14, [cq+64*14] |
| mova m15, [cq+64*15] |
| call .main_part1 |
| call .main_part2 |
| .pass1_end: |
| TRANSPOSE_4DQ 0, 4, 8, 12, 16 |
| TRANSPOSE_4DQ 1, 5, 9, 13, 16 |
| TRANSPOSE_4DQ 2, 6, 10, 14, 16 |
| TRANSPOSE_4DQ 3, 7, 11, 15, 16 |
| TRANSPOSE_4D 8, 9, 10, 11, 16 |
| TRANSPOSE_4D 12, 13, 14, 15, 16 |
| mov r6d, 64*12 |
| jmp .pass1_transpose_end |
| .pass1_fast: |
| WRAP_YMM IDCT16_PART1 |
| WRAP_YMM IDCT16_PART2 |
| .pass1_fast_end: |
| vinserti32x8 m0, ym4, 1 |
| vinserti32x8 m8, ym12, 1 |
| vinserti32x8 m1, ym5, 1 |
| vinserti32x8 m9, ym13, 1 |
| vinserti32x8 m2, ym6, 1 |
| vinserti32x8 m10, ym14, 1 |
| vinserti32x8 m3, ym7, 1 |
| vinserti32x8 m11, ym15, 1 |
| vshufi32x4 m4, m0, m8, q3131 |
| vshufi32x4 m0, m8, q2020 |
| vshufi32x4 m5, m1, m9, q3131 |
| vshufi32x4 m1, m9, q2020 |
| vshufi32x4 m6, m2, m10, q3131 |
| vshufi32x4 m2, m10, q2020 |
| vshufi32x4 m7, m3, m11, q3131 |
| vshufi32x4 m3, m11, q2020 |
| mov r6d, 64*4 |
| .pass1_transpose_end: |
| pxor m16, m16 |
| .zero_loop: |
| mova [cq+r6+64*0], m16 |
| mova [cq+r6+64*1], m16 |
| mova [cq+r6+64*2], m16 |
| mova [cq+r6+64*3], m16 |
| sub r6d, 64*4 |
| jge .zero_loop |
| TRANSPOSE_4D 0, 1, 2, 3, 16 |
| TRANSPOSE_4D 4, 5, 6, 7, 16 |
| jmp tx2q |
| .pass2: |
| test eobd, eobd |
| jl .pass2_fast |
| call .main_part1 |
| jmp .pass2_end |
| .pass2_fast: |
| call .main_part1_fast |
| .pass2_end: |
| vpbroadcastd m3, [o(pixel_clip6)] |
| paddd m0, m3 |
| paddd m18, m3 |
| call .main_part2 |
| REPX {psrad x, 6}, m0, m1, m2, m3 |
| packssdw m0, m1 |
| lea r6, [strideq*3] |
| packssdw m1, m2, m3 |
| mova m2, [o(deintq_perm)] |
| vpbroadcastd m3, [o(pixel_clip)] |
| REPX {psrad x, 6}, m4, m5, m6, m7 |
| call .write_16x4 |
| packssdw m0, m4, m5 |
| packssdw m1, m6, m7 |
| REPX {psrad x, 6}, m8, m9, m10, m11 |
| call .write_16x4 |
| packssdw m0, m8, m9 |
| packssdw m1, m10, m11 |
| .pass2_end2: |
| REPX {psrad x, 6}, m12, m13, m14, m15 |
| call .write_16x4 |
| packssdw m0, m12, m13 |
| packssdw m1, m14, m15 |
| call .write_16x4 |
| RET |
| ALIGN function_align |
| .write_16x4: |
| mova ym16, [dstq+strideq*0] |
| vinserti32x8 m16, [dstq+strideq*1], 1 |
| mova ym17, [dstq+strideq*2] |
| vinserti32x8 m17, [dstq+r6 ], 1 |
| vpermq m0, m2, m0 |
| vpermq m1, m2, m1 |
| paddsw m16, m0 |
| paddsw m17, m1 |
| psubusw m16, m3 |
| psubusw m17, m3 |
| mova [dstq+strideq*0], ym16 |
| vextracti32x8 [dstq+strideq*1], m16, 1 |
| mova [dstq+strideq*2], ym17 |
| vextracti32x8 [dstq+r6 ], m17, 1 |
| lea dstq, [dstq+strideq*4] |
| ret |
| ALIGN function_align |
| IDCT16_PART1 |
| ret |
| ALIGN function_align |
| .main_part2: |
| IDCT16_PART2 |
| ret |
| |
| %macro IADST16_PART1 0 |
| %if mmsize == 64 |
| .main_part1_fast: |
| %endif |
| pmulld m15, m0, [o(pd_16364)] {bcstd} ; t1 |
| pmulld m0, [o(pd_804)] {bcstd} ; t0 |
| pmulld m13, m2, [o(pd_15893)] {bcstd} ; t3 |
| pmulld m2, [o(pd_3981)] {bcstd} ; t2 |
| pmulld m11, m4, [o(pd_14811)] {bcstd} ; t5 |
| pmulld m4, [o(pd_7005)] {bcstd} ; t4 |
| pmulld m9, m6, [o(pd_13160)] {bcstd} ; t7 |
| pmulld m6, [o(pd_9760)] {bcstd} ; t6 |
| pmulld m8, m7, [o(pd_11003)] {bcstd} ; t8 |
| pmulld m7, [o(pd_12140)] {bcstd} ; t9 |
| pmulld m10, m5, [o(pd_8423)] {bcstd} ; t10 |
| pmulld m5, [o(pd_14053)] {bcstd} ; t11 |
| pmulld m12, m3, [o(pd_5520)] {bcstd} ; t12 |
| pmulld m3, [o(pd_15426)] {bcstd} ; t13 |
| pmulld m14, m1, [o(pd_2404)] {bcstd} ; t14 |
| pmulld m1, [o(pd_16207)] {bcstd} ; t15 |
| REPX {psubd x, m20, x}, m15, m13, m11, m9 |
| %if mmsize == 64 ; for the ymm variant we only ever use the fast path |
| jmp %%main_part1b |
| ALIGN function_align |
| .main_part1: |
| ITX_MULSUB_2D 15, 0, 16, 17, 18, _, 804, 16364 ; t1, t0 |
| ITX_MULSUB_2D 13, 2, 16, 17, 18, _, 3981, 15893 ; t3, t2 |
| ITX_MULSUB_2D 11, 4, 16, 17, 18, _, 7005, 14811 ; t5, t4 |
| ITX_MULSUB_2D 9, 6, 16, 17, 18, _, 9760, 13160 ; t7, t6 |
| ITX_MULSUB_2D 7, 8, 16, 17, 18, _, 12140, 11003 ; t9, t8 |
| ITX_MULSUB_2D 5, 10, 16, 17, 18, _, 14053, 8423 ; t11, t10 |
| ITX_MULSUB_2D 3, 12, 16, 17, 18, _, 15426, 5520 ; t13, t12 |
| ITX_MULSUB_2D 1, 14, 16, 17, 18, _, 16207, 2404 ; t15, t14 |
| REPX {paddd x, m20}, m15, m13, m11, m9 |
| %%main_part1b: |
| %endif |
| REPX {paddd x, m20}, m0, m2, m4, m6 |
| psubd m16, m2, m10 ; t10a |
| paddd m2, m10 ; t2a |
| psubd m10, m9, m1 ; t15a |
| paddd m9, m1 ; t7a |
| psubd m1, m13, m5 ; t11a |
| paddd m13, m5 ; t3a |
| psubd m5, m6, m14 ; t14a |
| paddd m6, m14 ; t6a |
| REPX {psrad x, 14}, m16, m10, m1, m5 |
| psubd m14, m0, m8 ; t8a |
| paddd m0, m8 ; t0a |
| psubd m8, m15, m7 ; t9a |
| paddd m15, m7 ; t1a |
| psubd m7, m4, m12 ; t12a |
| paddd m4, m12 ; t4a |
| paddd m12, m11, m3 ; t5a |
| psubd m11, m3 ; t13a |
| REPX {psrad x, 14}, m14, m8, m7, m11 |
| vpbroadcastd m19, [o(pd_9102)] |
| vpbroadcastd m18, [o(pd_13623)] |
| ITX_MULSUB_2D 16, 1, 3, 17, _, _, 18, 19 ; t11, t10 |
| ITX_MULSUB_2D 10, 5, 3, 17, _, _, 19, 18 ; t14, t15 |
| vpbroadcastd m19, [o(pd_16069)] |
| vpbroadcastd m18, [o(pd_3196)] |
| ITX_MULSUB_2D 14, 8, 3, 17, _, _, 18, 19 ; t9, t8 |
| ITX_MULSUB_2D 11, 7, 3, 17, _, _, 19, 18 ; t12, t13 |
| vpbroadcastd m19, [o(pd_6270)] |
| vpbroadcastd m18, [o(pd_15137)] |
| REPX {psrad x, 14}, m15, m12, m0, m4 |
| psubd m3, m15, m12 ; t5 |
| paddd m15, m12 ; t1 |
| psubd m12, m0, m4 ; t4 |
| paddd m0, m4 ; t0 |
| REPX {psrad x, 14}, m2, m6, m13, m9 |
| psubd m4, m2, m6 ; t6 |
| paddd m2, m6 ; t2 |
| psubd m6, m13, m9 ; t7 |
| paddd m9, m13 ; t3 |
| REPX {paddd x, m20}, m8, m14, m1, m16 |
| psubd m13, m8, m11 ; t12a |
| paddd m8, m11 ; t8a |
| psubd m11, m14, m7 ; t13a |
| paddd m14, m7 ; t9a |
| psubd m7, m1, m10 ; t14a |
| paddd m1, m10 ; t10a |
| psubd m10, m16, m5 ; t15a |
| paddd m16, m5 ; t11a |
| REPX {psrad x, 14}, m13, m11, m7, m10 |
| ITX_MULSUB_2D 12, 3, 5, 17, _, _, 19, 18 ; t5a, t4a |
| ITX_MULSUB_2D 6, 4, 5, 17, _, _, 18, 19 ; t6a, t7a |
| ITX_MULSUB_2D 13, 11, 5, 17, _, _, 19, 18 ; t13, t12 |
| ITX_MULSUB_2D 10, 7, 5, 17, _, _, 18, 19 ; t14, t15 |
| REPX {psrad x, 14}, m8, m1, m14, m16 |
| psubd m5, m8, m1 ; t10 |
| paddd m1, m8 ; -out1 |
| psubd m8, m15, m9 ; t3a |
| paddd m15, m9 ; -out15 |
| psubd m9, m14, m16 ; t11 |
| paddd m14, m16 ; out14 |
| psubd m16, m0, m2 ; t2a |
| paddd m0, m2 ; out0 |
| REPX {paddd x, m20}, m11, m13, m12, m3 |
| paddd m2, m11, m10 ; out2 |
| psubd m11, m10 ; t14a |
| psubd m10, m13, m7 ; t15a |
| paddd m13, m7 ; -out13 |
| psubd m7, m12, m4 ; t7 |
| paddd m12, m4 ; out12 |
| psubd m4, m3, m6 ; t6 |
| paddd m3, m6 ; -out3 |
| REPX {psrad x, 14}, m10, m7, m11, m4 |
| REPX {pmulld x, m21}, m9, m10, m7, m8, m5, m11, m4, m16 |
| REPX {psrad x, 14}, m2, m13, m12, m3 |
| %endmacro |
| |
| %macro IADST16_PART2 0 |
| paddd m9, m20 |
| psubd m10, m20, m10 |
| paddd m7, m20 |
| psubd m8, m20, m8 |
| paddd m6, m9, m5 ; out6 |
| psubd m9, m5 ; out9 |
| psubd m5, m10, m11 ; out5 |
| paddd m10, m11 ; out10 |
| psubd m11, m7, m4 ; out11 |
| paddd m4, m7 ; out4 |
| psubd m7, m8, m16 ; out7 |
| paddd m8, m16 ; out8 |
| %endmacro |
| |
| %macro IADST16_PASS1_END 0 |
| pxor m16, m16 |
| psubd m1, m16, m1 |
| psubd m3, m16, m3 |
| psubd m13, m16, m13 |
| psubd m15, m16, m15 |
| REPX {psrad x, 14}, m4, m5, m6, m7, m8, m9, m10, m11 |
| %endmacro |
| |
| INV_TXFM_16X16_FN adst, dct, 39-18 |
| INV_TXFM_16X16_FN adst, adst |
| |
| cglobal vp9_iadst_16x16_internal_10, 0, 7, 22, dst, stride, c, eob, tx2 |
| mova m0, [cq+64* 0] |
| mova m1, [cq+64* 1] |
| mova m2, [cq+64* 2] |
| mova m3, [cq+64* 3] |
| mova m4, [cq+64* 4] |
| mova m5, [cq+64* 5] |
| mova m6, [cq+64* 6] |
| mova m7, [cq+64* 7] |
| vpbroadcastd m20, [o(pd_8192)] |
| vpbroadcastd m21, [o(pd_11585)] |
| sub eobd, 39 |
| jl .pass1_fast |
| mova m8, [cq+64* 8] |
| mova m9, [cq+64* 9] |
| mova m10, [cq+64*10] |
| mova m11, [cq+64*11] |
| mova m12, [cq+64*12] |
| mova m13, [cq+64*13] |
| mova m14, [cq+64*14] |
| mova m15, [cq+64*15] |
| call .main_part1 |
| call .main_part2 |
| IADST16_PASS1_END |
| jmp m(vp9_idct_16x16_internal_10).pass1_end |
| .pass1_fast: |
| WRAP_YMM IADST16_PART1 |
| WRAP_YMM IADST16_PART2 |
| WRAP_YMM IADST16_PASS1_END |
| jmp m(vp9_idct_16x16_internal_10).pass1_fast_end |
| .pass2: |
| test eobd, eobd |
| jl .pass2_fast |
| call .main_part1 |
| jmp .pass2_end |
| .pass2_fast: |
| call .main_part1_fast |
| .pass2_end: |
| vpbroadcastd m20, [o(pd_532480)] |
| call .main_part2 |
| vpbroadcastd m16, [o(pixel_clip6)] |
| REPX {paddd x, m16}, m0, m2, m12, m14 |
| REPX {psubd x, m16, x}, m1, m3, m13, m15 |
| REPX {psrad x, 6}, m0, m1, m2, m3 |
| packssdw m0, m1 |
| lea r6, [strideq*3] |
| packssdw m1, m2, m3 |
| mova m2, [o(deintq_perm)] |
| vpbroadcastd m3, [o(pixel_clip)] |
| REPX {psrad x, 20}, m4, m5, m6, m7 |
| call m(vp9_idct_16x16_internal_10).write_16x4 |
| packssdw m0, m4, m5 |
| packssdw m1, m6, m7 |
| paddsw m0, m3 |
| paddsw m1, m3 |
| REPX {psrad x, 20}, m8, m9, m10, m11 |
| call m(vp9_idct_16x16_internal_10).write_16x4 |
| packssdw m0, m8, m9 |
| packssdw m1, m10, m11 |
| paddsw m0, m3 |
| paddsw m1, m3 |
| jmp m(vp9_idct_16x16_internal_10).pass2_end2 |
| ALIGN function_align |
| IADST16_PART1 |
| ret |
| ALIGN function_align |
| .main_part2: |
| IADST16_PART2 |
| ret |
| |
| cglobal vp9_idct_idct_32x32_add_10, 4, 7, 23, 64*64, dst, stride, c, eob |
| %undef cmp |
| lea r5, [o_base] |
| dec eobd |
| jnz .pass1 |
| imul r6d, [cq], 11585 |
| vpbroadcastd m3, [o(pixel_clip)] |
| mov [cq], r3d |
| add r6d, 8192 |
| sar r6d, 14 |
| imul r6d, 11585 |
| or r3d, 16 |
| add r6d, 532480 |
| sar r6d, 20 |
| vpbroadcastw m2, r6d |
| paddsw m2, m3 |
| .dconly_loop: |
| paddsw m0, m2, [dstq+strideq*0] |
| paddsw m1, m2, [dstq+strideq*1] |
| psubusw m0, m3 |
| psubusw m1, m3 |
| mova [dstq+strideq*0], m0 |
| mova [dstq+strideq*1], m1 |
| lea dstq, [dstq+strideq*2] |
| dec r3d |
| jg .dconly_loop |
| RET |
| .pass1: |
| vpbroadcastd m20, [o(pd_8192)] |
| vpbroadcastd m21, [o(pd_11585)] |
| cmp eobd, 135 |
| jl .pass1_fast |
| add cq, 64 |
| lea r4, [rsp+64*8] |
| cmp eobd, 579 |
| jl .pass1_right_fast |
| mov r6d, 128*28 |
| call .pass1_main |
| jmp .pass1_right_end |
| .pass1_right_fast: ; bottomright quadrant is zero |
| mova m0, [cq+128* 1] |
| mova m1, [cq+128* 3] |
| mova m2, [cq+128* 5] |
| mova m3, [cq+128* 7] |
| mova m4, [cq+128* 9] |
| mova m5, [cq+128*11] |
| mova m6, [cq+128*13] |
| mova m7, [cq+128*15] |
| call .main_fast |
| mova m0, [cq+128* 0] |
| mova m1, [cq+128* 2] |
| mova m2, [cq+128* 4] |
| mova m3, [cq+128* 6] |
| mova m4, [cq+128* 8] |
| mova m5, [cq+128*10] |
| mova m6, [cq+128*12] |
| mova m7, [cq+128*14] |
| call m(vp9_idct_16x16_internal_10).main_part1_fast |
| mov r6d, 128*12 |
| call .pass1_main_end |
| .pass1_right_end: |
| mova [r4+64* 8], m0 |
| mova [r4+64* 9], m1 |
| mova [r4+64*10], m2 |
| mova [r4+64*11], m3 |
| mova [r4+64*12], m4 |
| mova [r4+64*13], m5 |
| mova [r4+64*14], m6 |
| mova [r4+64*15], m7 |
| mova [r4+64*16], m16 |
| mova [r4+64*17], m17 |
| mova [r4+64*18], m18 |
| mova [r4+64*19], m19 |
| mova [r4+64*20], m8 |
| mova [r4+64*21], m9 |
| mova [r4+64*22], m10 |
| mova [r4+64*23], m11 |
| sub cq, 64 |
| sub r4, 64*8 |
| mov r6d, 128*28 |
| call .pass1_main |
| mova m12, [r4+64*20] |
| mova m13, [r4+64*21] |
| mova m14, [r4+64*22] |
| mova m15, [r4+64*23] |
| mova [r4+64*20], m8 |
| mova [r4+64*21], m9 |
| mova [r4+64*22], m10 |
| mova [r4+64*23], m11 |
| mova m8, [r4+64*16] |
| mova m9, [r4+64*17] |
| mova m10, [r4+64*18] |
| mova m11, [r4+64*19] |
| mova [r4+64*16], m16 |
| mova [r4+64*17], m17 |
| mova [r4+64*18], m18 |
| mova [r4+64*19], m19 |
| call .main |
| mova m0, [r4+64*16] |
| mova m1, [r4+64*17] |
| mova m2, [r4+64*18] |
| mova m3, [r4+64*19] |
| mova m4, [r4+64*20] |
| mova m5, [r4+64*21] |
| mova m6, [r4+64*22] |
| mova m7, [r4+64*23] |
| mova m8, [r4+64*24] |
| mova m9, [r4+64*25] |
| mova m10, [r4+64*26] |
| mova m11, [r4+64*27] |
| mova m12, [r4+64*28] |
| mova m13, [r4+64*29] |
| mova m14, [r4+64*30] |
| mova m15, [r4+64*31] |
| call m(vp9_idct_16x16_internal_10).main_part1 |
| call .pass2_main_left |
| mova m8, [r4+64* 8] |
| mova m9, [r4+64* 9] |
| mova m10, [r4+64*10] |
| mova m11, [r4+64*11] |
| mova m12, [r4+64*12] |
| mova m13, [r4+64*13] |
| mova m14, [r4+64*14] |
| mova m15, [r4+64*15] |
| TRANSPOSE_4DQ 8, 10, 12, 14, 16 |
| TRANSPOSE_4DQ 9, 11, 13, 15, 16 |
| call .main |
| call .pass2_main_right |
| mova m8, [r4+64*24] |
| mova m9, [r4+64*25] |
| mova m10, [r4+64*26] |
| mova m11, [r4+64*27] |
| mova m12, [r4+64*28] |
| mova m13, [r4+64*29] |
| mova m14, [r4+64*30] |
| mova m15, [r4+64*31] |
| TRANSPOSE_4DQ 8, 10, 12, 14, 16 |
| TRANSPOSE_4DQ 9, 11, 13, 15, 16 |
| call m(vp9_idct_16x16_internal_10).main_part1 |
| jmp .pass2_end |
| .pass1_fast: |
| mova m0, [cq+128* 1] |
| mova m1, [cq+128* 3] |
| mova m2, [cq+128* 5] |
| mova m3, [cq+128* 7] |
| mova m4, [cq+128* 9] |
| mova m5, [cq+128*11] |
| mova m6, [cq+128*13] |
| mova m7, [cq+128*15] |
| mov r4, rsp |
| call .main_fast |
| mova m0, [cq+128* 0] |
| mova m1, [cq+128* 2] |
| mova m2, [cq+128* 4] |
| mova m3, [cq+128* 6] |
| mova m4, [cq+128* 8] |
| mova m5, [cq+128*10] |
| mova m6, [cq+128*12] |
| mova m7, [cq+128*14] |
| call m(vp9_idct_16x16_internal_10).main_part1_fast |
| call m(vp9_idct_16x16_internal_10).main_part2 |
| mov r6d, 128*12 |
| call .pass1_main_end2 |
| mova [r4+64*16], m16 |
| mova [r4+64*17], m17 |
| mova [r4+64*18], m18 |
| mova [r4+64*19], m19 |
| mova [r4+64*20], m8 |
| mova [r4+64*21], m9 |
| mova [r4+64*22], m10 |
| mova [r4+64*23], m11 |
| call .main_fast |
| mova m0, [r4+64*16] |
| mova m1, [r4+64*17] |
| mova m2, [r4+64*18] |
| mova m3, [r4+64*19] |
| mova m4, [r4+64*20] |
| mova m5, [r4+64*21] |
| mova m6, [r4+64*22] |
| mova m7, [r4+64*23] |
| call m(vp9_idct_16x16_internal_10).main_part1_fast |
| call .pass2_main_left |
| call .main_fast |
| call .pass2_main_right |
| call m(vp9_idct_16x16_internal_10).main_part1_fast |
| .pass2_end: |
| paddd m0, m22 |
| paddd m18, m22 |
| call m(vp9_idct_16x16_internal_10).main_part2 |
| mova m20, [o(deintq_perm)] |
| rorx r2, strideq, 59 ; strideq*32 |
| vpbroadcastd m21, [o(pixel_clip)] |
| add r2, dstq |
| %assign i 0 |
| %rep 16 |
| mova m16, [r4+64*(15-i)] |
| mova m17, [r4+64*(i-16)] |
| mova m18, [r4-64*(17+i)] |
| paddd m19, m %+ i, m16 |
| psubd m0, m %+ i, m16 |
| call .write_32x2 |
| %assign i i+1 |
| %endrep |
| RET |
| ALIGN function_align |
| .write_32x2: |
| paddd m16, m17, m18 |
| psubd m17, m18 |
| REPX {psrad x, 6}, m19, m16, m0, m17 |
| packssdw m16, m19 |
| packssdw m17, m0 |
| sub r2, strideq |
| vpermq m16, m20, m16 |
| vpermq m17, m20, m17 |
| paddsw m16, [dstq] |
| paddsw m17, [r2 ] |
| psubusw m16, m21 |
| psubusw m17, m21 |
| mova [dstq], m16 |
| mova [r2 ], m17 |
| add dstq, strideq |
| ret |
| ALIGN function_align |
| .pass1_main: |
| mova m0, [cq+128* 1] |
| mova m1, [cq+128* 3] |
| mova m2, [cq+128* 5] |
| mova m3, [cq+128* 7] |
| mova m4, [cq+128* 9] |
| mova m5, [cq+128*11] |
| mova m6, [cq+128*13] |
| mova m7, [cq+128*15] |
| mova m8, [cq+128*17] |
| mova m9, [cq+128*19] |
| mova m10, [cq+128*21] |
| mova m11, [cq+128*23] |
| mova m12, [cq+128*25] |
| mova m13, [cq+128*27] |
| mova m14, [cq+128*29] |
| mova m15, [cq+128*31] |
| call .main |
| mova m0, [cq+128* 0] |
| mova m1, [cq+128* 2] |
| mova m2, [cq+128* 4] |
| mova m3, [cq+128* 6] |
| mova m4, [cq+128* 8] |
| mova m5, [cq+128*10] |
| mova m6, [cq+128*12] |
| mova m7, [cq+128*14] |
| mova m8, [cq+128*16] |
| mova m9, [cq+128*18] |
| mova m10, [cq+128*20] |
| mova m11, [cq+128*22] |
| mova m12, [cq+128*24] |
| mova m13, [cq+128*26] |
| mova m14, [cq+128*28] |
| mova m15, [cq+128*30] |
| call m(vp9_idct_16x16_internal_10).main_part1 |
| .pass1_main_end: |
| call m(vp9_idct_16x16_internal_10).main_part2 |
| .pass1_main_end2: |
| pxor m16, m16 |
| .pass1_zero_loop: |
| mova [cq+r6+128*0], m16 |
| mova [cq+r6+128*1], m16 |
| mova [cq+r6+128*2], m16 |
| mova [cq+r6+128*3], m16 |
| sub r6d, 128*4 |
| jge .pass1_zero_loop |
| mova m16, [r4+64*15] |
| mova m19, [r4+64*14] |
| mova m22, [r4+64*13] |
| mova m17, [r4+64*12] |
| psubd m18, m0, m16 |
| paddd m16, m0 |
| paddd m0, m19, m1 |
| psubd m19, m1, m19 |
| paddd m1, m17, m3 |
| psubd m3, m17 |
| paddd m17, m2, m22 |
| psubd m2, m22 |
| TRANSPOSE_4D 3, 2, 19, 18, 22 ; 28 29 30 31 |
| TRANSPOSE_4D 16, 0, 17, 1, 22 ; 0 1 2 3 |
| mova [r4+64*54], m3 |
| mova [r4+64*55], m19 |
| mova [r4+64*38], m2 |
| mova [r4+64*39], m18 |
| mova m2, [r4+64*11] |
| mova m19, [r4+64*10] |
| mova m3, [r4+64* 9] |
| mova m22, [r4+64* 8] |
| paddd m18, m4, m2 |
| psubd m4, m2 |
| paddd m2, m5, m19 |
| psubd m5, m19 |
| paddd m19, m6, m3 |
| psubd m6, m3 |
| paddd m3, m7, m22 |
| psubd m7, m22 |
| TRANSPOSE_4D 7, 6, 5, 4, 22 ; 24 25 26 27 |
| TRANSPOSE_4D 18, 2, 19, 3, 22 ; 4 5 6 7 |
| mova [r4+64*52], m7 |
| mova [r4+64*53], m5 |
| mova [r4+64*36], m6 |
| mova [r4+64*37], m4 |
| mova m7, [r4+64* 7] |
| mova m4, [r4+64* 6] |
| mova m5, [r4+64* 5] |
| mova m22, [r4+64* 4] |
| psubd m6, m8, m7 |
| paddd m8, m7 |
| psubd m7, m9, m4 |
| paddd m4, m9 |
| paddd m9, m10, m5 |
| psubd m10, m5 |
| paddd m5, m11, m22 |
| psubd m11, m22 |
| TRANSPOSE_4D 11, 10, 7, 6, 22 ; 20 21 22 23 |
| TRANSPOSE_4D 8, 4, 9, 5, 22 ; 8 9 10 11 |
| mova [r4+64*50], m11 |
| mova [r4+64*51], m7 |
| mova [r4+64*34], m10 |
| mova [r4+64*35], m6 |
| mova m6, [r4+64* 3] |
| mova m11, [r4+64* 2] |
| mova m7, [r4+64* 1] |
| mova m22, [r4+64* 0] |
| paddd m10, m12, m6 |
| psubd m12, m6 |
| paddd m6, m13, m11 |
| psubd m13, m11 |
| paddd m11, m14, m7 |
| psubd m14, m7 |
| paddd m7, m15, m22 |
| psubd m15, m22 |
| TRANSPOSE_4D 15, 14, 13, 12, 22 ; 16 17 18 19 |
| TRANSPOSE_4D 10, 6, 11, 7, 22 ; 12 13 14 15 |
| mova [r4+64*48], m15 |
| mova [r4+64*49], m13 |
| mova [r4+64*32], m14 |
| mova [r4+64*33], m12 |
| TRANSPOSE_4DQ 0, 2, 4, 6, 22 |
| TRANSPOSE_4DQ 1, 3, 5, 7, 22 |
| TRANSPOSE_4DQ 16, 18, 8, 10, 22 |
| TRANSPOSE_4DQ 17, 19, 9, 11, 22 |
| ret |
| ALIGN function_align |
| .pass2_main_left: |
| vpbroadcastd m22, [o(pixel_clip6)] |
| paddd m0, m22 |
| paddd m18, m22 |
| call m(vp9_idct_16x16_internal_10).main_part2 |
| mova [r4+64*16], m0 |
| mova [r4+64*17], m1 |
| mova [r4+64*18], m2 |
| mova [r4+64*19], m3 |
| mova [r4+64*20], m4 |
| mova [r4+64*21], m5 |
| mova [r4+64*22], m6 |
| mova [r4+64*23], m7 |
| mova [r4+64*24], m8 |
| mova [r4+64*25], m9 |
| mova [r4+64*26], m10 |
| mova [r4+64*27], m11 |
| mova [r4+64*28], m12 |
| mova [r4+64*29], m13 |
| mova [r4+64*30], m14 |
| mova [r4+64*31], m15 |
| add r4, 64*32 |
| mova m0, [r4+64* 0] |
| mova m1, [r4+64* 1] |
| mova m2, [r4+64* 2] |
| mova m3, [r4+64* 3] |
| mova m4, [r4+64* 4] |
| mova m5, [r4+64* 5] |
| mova m6, [r4+64* 6] |
| mova m7, [r4+64* 7] |
| jmp .pass2_main_transpose |
| ALIGN function_align |
| .pass2_main_right: |
| mova m0, [r4+64*16] |
| mova m1, [r4+64*17] |
| mova m2, [r4+64*18] |
| mova m3, [r4+64*19] |
| mova m4, [r4+64*20] |
| mova m5, [r4+64*21] |
| mova m6, [r4+64*22] |
| mova m7, [r4+64*23] |
| .pass2_main_transpose: |
| TRANSPOSE_4DQ 0, 2, 4, 6, 8 |
| TRANSPOSE_4DQ 1, 3, 5, 7, 8 |
| ret |
| ALIGN function_align |
| .main_fast: |
| pmulld m15, m0, [o(pd_16364)] {1to16} ; t31a |
| pmulld m0, [o(pd_804)] {1to16} ; t16a |
| pmulld m8, m7, [o(pd_11003)] {1to16} ; t17a |
| pmulld m7, [o(pd_12140)] {1to16} ; t30a |
| pmulld m11, m4, [o(pd_14811)] {1to16} ; t29a |
| pmulld m4, [o(pd_7005)] {1to16} ; t18a |
| pmulld m12, m3, [o(pd_5520)] {1to16} ; t19a |
| pmulld m3, [o(pd_15426)] {1to16} ; t28a |
| pmulld m13, m2, [o(pd_15893)] {1to16} ; t27a |
| pmulld m2, [o(pd_3981)] {1to16} ; t20a |
| pmulld m10, m5, [o(pd_8423)] {1to16} ; t21a |
| pmulld m5, [o(pd_14053)] {1to16} ; t26a |
| pmulld m9, m6, [o(pd_13160)] {1to16} ; t25a |
| pmulld m6, [o(pd_9760)] {1to16} ; t22a |
| pmulld m14, m1, [o(pd_2404)] {1to16} ; t23a |
| pmulld m1, [o(pd_16207)] {1to16} ; t24a |
| REPX {psubd x, m20, x}, m8, m12, m10, m14 |
| jmp .main2 |
| ALIGN function_align |
| .main: |
| ITX_MULSUB_2D 0, 15, 16, 17, 18, _, 804, 16364 ; t16a, t31a |
| ITX_MULSUB_2D 8, 7, 16, 17, 18, _, 12140, 11003 ; t17a, t30a |
| ITX_MULSUB_2D 4, 11, 16, 17, 18, _, 7005, 14811 ; t18a, t29a |
| ITX_MULSUB_2D 12, 3, 16, 17, 18, _, 15426, 5520 ; t19a, t28a |
| ITX_MULSUB_2D 2, 13, 16, 17, 18, _, 3981, 15893 ; t20a, t27a |
| ITX_MULSUB_2D 10, 5, 16, 17, 18, _, 14053, 8423 ; t21a, t26a |
| ITX_MULSUB_2D 6, 9, 16, 17, 18, _, 9760, 13160 ; t22a, t25a |
| ITX_MULSUB_2D 14, 1, 16, 17, 18, _, 16207, 2404 ; t23a, t24a |
| REPX {paddd x, m20}, m8, m12, m10, m14 |
| .main2: |
| REPX {paddd x, m20}, m0, m15, m7, m4, m3, m11 |
| REPX {psrad x, 14 }, m8, m0, m15, m7, m12, m4, m3, m11 |
| psubd m16, m0, m8 ; t17 |
| paddd m0, m8 ; t16 |
| psubd m8, m15, m7 ; t30 |
| paddd m15, m7 ; t31 |
| paddd m7, m12, m4 ; t19 |
| psubd m12, m4 ; t18 |
| paddd m4, m3, m11 ; t28 |
| psubd m3, m11 ; t29 |
| REPX {paddd x, m20}, m2, m13, m5, m6, m1, m9 |
| REPX {psrad x, 14 }, m10, m2, m13, m5, m14, m6, m1, m9 |
| psubd m11, m2, m10 ; t21 |
| paddd m2, m10 ; t20 |
| psubd m10, m13, m5 ; t26 |
| paddd m13, m5 ; t27 |
| psubd m5, m14, m6 ; t22 |
| paddd m6, m14 ; t23 |
| psubd m14, m1, m9 ; t25 |
| paddd m9, m1 ; t24 |
| vpbroadcastd m19, [o(pd_16069)] |
| vpbroadcastd m18, [o(pd_3196)] |
| ITX_MULSUB_2D 8, 16, 1, 17, _, 20, 18, 19 ; t17a, t30a |
| ITX_MULSUB_2D 3, 12, 1, 17, _, 20, 18, 19, 1 ; t29a, t18a |
| vpbroadcastd m19, [o(pd_9102)] |
| vpbroadcastd m18, [o(pd_13623)] |
| ITX_MULSUB_2D 10, 11, 1, 17, _, 20, 18, 19 ; t21a, t26a |
| ITX_MULSUB_2D 14, 5, 1, 17, _, 20, 18, 19, 1 ; t25a, t22a |
| paddd m1, m6, m2 ; t23a |
| psubd m6, m2 ; t20a |
| psubd m2, m9, m13 ; t27a |
| paddd m9, m13 ; t24a |
| psubd m13, m15, m4 ; t28a |
| paddd m15, m4 ; t31a |
| psubd m4, m8, m12 ; t18 |
| paddd m8, m12 ; t17 |
| psubd m12, m0, m7 ; t19a |
| paddd m0, m7 ; t16a |
| psubd m7, m16, m3 ; t29 |
| paddd m3, m16 ; t30 |
| paddd m16, m5, m10 ; t22 |
| psubd m5, m10 ; t21 |
| psubd m10, m14, m11 ; t26 |
| paddd m14, m11 ; t25 |
| vpbroadcastd m19, [o(pd_15137)] |
| vpbroadcastd m18, [o(pd_6270)] |
| ITX_MULSUB_2D 13, 12, 11, 17, _, 20, 18, 19 ; t19, t28 |
| ITX_MULSUB_2D 2, 6, 11, 17, _, 20, 18, 19, 1 ; t27, t20 |
| ITX_MULSUB_2D 7, 4, 11, 17, _, 20, 18, 19 ; t18a, t29a |
| ITX_MULSUB_2D 10, 5, 11, 17, _, 20, 18, 19, 1 ; t26a, t21a |
| psubd m11, m0, m1 ; t23 |
| paddd m0, m1 ; t16 |
| paddd m1, m16, m8 ; t17a |
| psubd m16, m8, m16 ; t22a |
| psubd m8, m15, m9 ; t24 |
| paddd m15, m9 ; t31 |
| psubd m9, m3, m14 ; t25a |
| paddd m14, m3 ; t30a |
| paddd m3, m6, m13 ; t19a |
| psubd m6, m13, m6 ; t20a |
| paddd m13, m10, m4 ; t29 |
| psubd m10, m4, m10 ; t26 |
| psubd m4, m12, m2 ; t27a |
| paddd m12, m2 ; t28a |
| paddd m2, m7, m5 ; t18 |
| psubd m7, m5 ; t21 |
| REPX {pmulld x, m21}, m10, m8, m4, m9, m7, m11, m6, m16 |
| mova [r4+64* 0], m0 |
| mova [r4+64* 1], m1 |
| mova [r4+64* 2], m2 |
| mova [r4+64* 3], m3 |
| mova [r4+64*12], m12 |
| mova [r4+64*13], m13 |
| mova [r4+64*14], m14 |
| mova [r4+64*15], m15 |
| REPX {paddd x, m20}, m10, m8, m4, m9 |
| psubd m5, m10, m7 ; t21a |
| paddd m10, m7 ; t26a |
| psubd m7, m8, m11 ; t23a |
| paddd m8, m11 ; t24a |
| REPX {psrad x, 14 }, m5, m10, m7, m8 |
| paddd m11, m4, m6 ; t27 |
| psubd m4, m6 ; t20 |
| psubd m6, m9, m16 ; t22 |
| paddd m9, m16 ; t25 |
| REPX {psrad x, 14 }, m11, m4, m6, m9 |
| mova [r4+64* 4], m4 |
| mova [r4+64* 5], m5 |
| mova [r4+64* 6], m6 |
| mova [r4+64* 7], m7 |
| mova [r4+64* 8], m8 |
| mova [r4+64* 9], m9 |
| mova [r4+64*10], m10 |
| mova [r4+64*11], m11 |
| ret |
| |
| %endif |
