| /* |
| * Copyright (c) 2021 Loongson Technology Corporation Limited |
| * Contributed by Hao Chen <chenhao@loongson.cn> |
| * |
| * This file is part of FFmpeg. |
| * |
| * FFmpeg is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU Lesser General Public |
| * License as published by the Free Software Foundation; either |
| * version 2.1 of the License, or (at your option) any later version. |
| * |
| * FFmpeg is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| * Lesser General Public License for more details. |
| * |
| * You should have received a copy of the GNU Lesser General Public |
| * License along with FFmpeg; if not, write to the Free Software |
| * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| */ |
| |
| #include "libavcodec/vp9dsp.h" |
| #include "libavutil/loongarch/loongson_intrinsics.h" |
| #include "vp9dsp_loongarch.h" |
| |
| #define LSX_ST_8(_dst0, _dst1, _dst2, _dst3, _dst4, \ |
| _dst5, _dst6, _dst7, _dst, _stride, \ |
| _stride2, _stride3, _stride4) \ |
| { \ |
| __lsx_vst(_dst0, _dst, 0); \ |
| __lsx_vstx(_dst1, _dst, _stride); \ |
| __lsx_vstx(_dst2, _dst, _stride2); \ |
| __lsx_vstx(_dst3, _dst, _stride3); \ |
| _dst += _stride4; \ |
| __lsx_vst(_dst4, _dst, 0); \ |
| __lsx_vstx(_dst5, _dst, _stride); \ |
| __lsx_vstx(_dst6, _dst, _stride2); \ |
| __lsx_vstx(_dst7, _dst, _stride3); \ |
| } |
| |
| #define LSX_ST_8X16(_dst0, _dst1, _dst2, _dst3, _dst4, \ |
| _dst5, _dst6, _dst7, _dst, _stride) \ |
| { \ |
| __lsx_vst(_dst0, _dst, 0); \ |
| __lsx_vst(_dst0, _dst, 16); \ |
| _dst += _stride; \ |
| __lsx_vst(_dst1, _dst, 0); \ |
| __lsx_vst(_dst1, _dst, 16); \ |
| _dst += _stride; \ |
| __lsx_vst(_dst2, _dst, 0); \ |
| __lsx_vst(_dst2, _dst, 16); \ |
| _dst += _stride; \ |
| __lsx_vst(_dst3, _dst, 0); \ |
| __lsx_vst(_dst3, _dst, 16); \ |
| _dst += _stride; \ |
| __lsx_vst(_dst4, _dst, 0); \ |
| __lsx_vst(_dst4, _dst, 16); \ |
| _dst += _stride; \ |
| __lsx_vst(_dst5, _dst, 0); \ |
| __lsx_vst(_dst5, _dst, 16); \ |
| _dst += _stride; \ |
| __lsx_vst(_dst6, _dst, 0); \ |
| __lsx_vst(_dst6, _dst, 16); \ |
| _dst += _stride; \ |
| __lsx_vst(_dst7, _dst, 0); \ |
| __lsx_vst(_dst7, _dst, 16); \ |
| _dst += _stride; \ |
| } |
| |
| void ff_vert_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *left, |
| const uint8_t *src) |
| { |
| __m128i src0; |
| ptrdiff_t stride2 = dst_stride << 1; |
| ptrdiff_t stride3 = stride2 + dst_stride; |
| ptrdiff_t stride4 = stride2 << 1; |
| src0 = __lsx_vld(src, 0); |
| LSX_ST_8(src0, src0, src0, src0, src0, src0, src0, src0, dst, |
| dst_stride, stride2, stride3, stride4); |
| dst += stride4; |
| LSX_ST_8(src0, src0, src0, src0, src0, src0, src0, src0, dst, |
| dst_stride, stride2, stride3, stride4); |
| } |
| |
| void ff_vert_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *left, |
| const uint8_t *src) |
| { |
| uint32_t row; |
| __m128i src0, src1; |
| |
| DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); |
| for (row = 32; row--;) { |
| __lsx_vst(src0, dst, 0); |
| __lsx_vst(src1, dst, 16); |
| dst += dst_stride; |
| } |
| } |
| |
| void ff_hor_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src, |
| const uint8_t *top) |
| { |
| __m128i src0, src1, src2, src3, src4, src5, src6, src7; |
| __m128i src8, src9, src10, src11, src12, src13, src14, src15; |
| ptrdiff_t stride2 = dst_stride << 1; |
| ptrdiff_t stride3 = stride2 + dst_stride; |
| ptrdiff_t stride4 = stride2 << 1; |
| |
| src15 = __lsx_vldrepl_b(src, 0); |
| src14 = __lsx_vldrepl_b(src, 1); |
| src13 = __lsx_vldrepl_b(src, 2); |
| src12 = __lsx_vldrepl_b(src, 3); |
| src11 = __lsx_vldrepl_b(src, 4); |
| src10 = __lsx_vldrepl_b(src, 5); |
| src9 = __lsx_vldrepl_b(src, 6); |
| src8 = __lsx_vldrepl_b(src, 7); |
| src7 = __lsx_vldrepl_b(src, 8); |
| src6 = __lsx_vldrepl_b(src, 9); |
| src5 = __lsx_vldrepl_b(src, 10); |
| src4 = __lsx_vldrepl_b(src, 11); |
| src3 = __lsx_vldrepl_b(src, 12); |
| src2 = __lsx_vldrepl_b(src, 13); |
| src1 = __lsx_vldrepl_b(src, 14); |
| src0 = __lsx_vldrepl_b(src, 15); |
| LSX_ST_8(src0, src1, src2, src3, src4, src5, src6, src7, dst, |
| dst_stride, stride2, stride3, stride4); |
| dst += stride4; |
| LSX_ST_8(src8, src9, src10, src11, src12, src13, src14, src15, dst, |
| dst_stride, stride2, stride3, stride4); |
| } |
| |
| void ff_hor_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src, |
| const uint8_t *top) |
| { |
| __m128i src0, src1, src2, src3, src4, src5, src6, src7; |
| __m128i src8, src9, src10, src11, src12, src13, src14, src15; |
| __m128i src16, src17, src18, src19, src20, src21, src22, src23; |
| __m128i src24, src25, src26, src27, src28, src29, src30, src31; |
| |
| src31 = __lsx_vldrepl_b(src, 0); |
| src30 = __lsx_vldrepl_b(src, 1); |
| src29 = __lsx_vldrepl_b(src, 2); |
| src28 = __lsx_vldrepl_b(src, 3); |
| src27 = __lsx_vldrepl_b(src, 4); |
| src26 = __lsx_vldrepl_b(src, 5); |
| src25 = __lsx_vldrepl_b(src, 6); |
| src24 = __lsx_vldrepl_b(src, 7); |
| src23 = __lsx_vldrepl_b(src, 8); |
| src22 = __lsx_vldrepl_b(src, 9); |
| src21 = __lsx_vldrepl_b(src, 10); |
| src20 = __lsx_vldrepl_b(src, 11); |
| src19 = __lsx_vldrepl_b(src, 12); |
| src18 = __lsx_vldrepl_b(src, 13); |
| src17 = __lsx_vldrepl_b(src, 14); |
| src16 = __lsx_vldrepl_b(src, 15); |
| src15 = __lsx_vldrepl_b(src, 16); |
| src14 = __lsx_vldrepl_b(src, 17); |
| src13 = __lsx_vldrepl_b(src, 18); |
| src12 = __lsx_vldrepl_b(src, 19); |
| src11 = __lsx_vldrepl_b(src, 20); |
| src10 = __lsx_vldrepl_b(src, 21); |
| src9 = __lsx_vldrepl_b(src, 22); |
| src8 = __lsx_vldrepl_b(src, 23); |
| src7 = __lsx_vldrepl_b(src, 24); |
| src6 = __lsx_vldrepl_b(src, 25); |
| src5 = __lsx_vldrepl_b(src, 26); |
| src4 = __lsx_vldrepl_b(src, 27); |
| src3 = __lsx_vldrepl_b(src, 28); |
| src2 = __lsx_vldrepl_b(src, 29); |
| src1 = __lsx_vldrepl_b(src, 30); |
| src0 = __lsx_vldrepl_b(src, 31); |
| LSX_ST_8X16(src0, src1, src2, src3, src4, src5, src6, src7, |
| dst, dst_stride); |
| LSX_ST_8X16(src8, src9, src10, src11, src12, src13, src14, src15, |
| dst, dst_stride); |
| LSX_ST_8X16(src16, src17, src18, src19, src20, src21, src22, src23, |
| dst, dst_stride); |
| LSX_ST_8X16(src24, src25, src26, src27, src28, src29, src30, src31, |
| dst, dst_stride); |
| } |
| |
| void ff_dc_4x4_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left, |
| const uint8_t *src_top) |
| { |
| __m128i tmp0, tmp1, dst0; |
| |
| tmp0 = __lsx_vldrepl_w(src_top, 0); |
| tmp1 = __lsx_vldrepl_w(src_left, 0); |
| dst0 = __lsx_vilvl_w(tmp1, tmp0); |
| dst0 = __lsx_vhaddw_hu_bu(dst0, dst0); |
| dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); |
| dst0 = __lsx_vhaddw_du_wu(dst0, dst0); |
| dst0 = __lsx_vsrari_w(dst0, 3); |
| dst0 = __lsx_vshuf4i_b(dst0, 0); |
| __lsx_vstelm_w(dst0, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_w(dst0, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_w(dst0, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_w(dst0, dst, 0, 0); |
| } |
| |
| #define INTRA_DC_TL_4X4(dir) \ |
| void ff_dc_##dir##_4x4_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ |
| const uint8_t *left, \ |
| const uint8_t *top) \ |
| { \ |
| __m128i tmp0, dst0; \ |
| \ |
| tmp0 = __lsx_vldrepl_w(dir, 0); \ |
| dst0 = __lsx_vhaddw_hu_bu(tmp0, tmp0); \ |
| dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); \ |
| dst0 = __lsx_vsrari_w(dst0, 2); \ |
| dst0 = __lsx_vshuf4i_b(dst0, 0); \ |
| __lsx_vstelm_w(dst0, dst, 0, 0); \ |
| dst += dst_stride; \ |
| __lsx_vstelm_w(dst0, dst, 0, 0); \ |
| dst += dst_stride; \ |
| __lsx_vstelm_w(dst0, dst, 0, 0); \ |
| dst += dst_stride; \ |
| __lsx_vstelm_w(dst0, dst, 0, 0); \ |
| } |
| INTRA_DC_TL_4X4(top); |
| INTRA_DC_TL_4X4(left); |
| |
| void ff_dc_8x8_lsx(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src_left, |
| const uint8_t *src_top) |
| { |
| __m128i tmp0, tmp1, dst0; |
| |
| tmp0 = __lsx_vldrepl_d(src_top, 0); |
| tmp1 = __lsx_vldrepl_d(src_left, 0); |
| dst0 = __lsx_vilvl_d(tmp1, tmp0); |
| dst0 = __lsx_vhaddw_hu_bu(dst0, dst0); |
| dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); |
| dst0 = __lsx_vhaddw_du_wu(dst0, dst0); |
| dst0 = __lsx_vhaddw_qu_du(dst0, dst0); |
| dst0 = __lsx_vsrari_w(dst0, 4); |
| dst0 = __lsx_vreplvei_b(dst0, 0); |
| __lsx_vstelm_d(dst0, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_d(dst0, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_d(dst0, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_d(dst0, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_d(dst0, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_d(dst0, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_d(dst0, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_d(dst0, dst, 0, 0); |
| } |
| |
| #define INTRA_DC_TL_8X8(dir) \ |
| void ff_dc_##dir##_8x8_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ |
| const uint8_t *left, \ |
| const uint8_t *top) \ |
| { \ |
| __m128i tmp0, dst0; \ |
| \ |
| tmp0 = __lsx_vldrepl_d(dir, 0); \ |
| dst0 = __lsx_vhaddw_hu_bu(tmp0, tmp0); \ |
| dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); \ |
| dst0 = __lsx_vhaddw_du_wu(dst0, dst0); \ |
| dst0 = __lsx_vsrari_w(dst0, 3); \ |
| dst0 = __lsx_vreplvei_b(dst0, 0); \ |
| __lsx_vstelm_d(dst0, dst, 0, 0); \ |
| dst += dst_stride; \ |
| __lsx_vstelm_d(dst0, dst, 0, 0); \ |
| dst += dst_stride; \ |
| __lsx_vstelm_d(dst0, dst, 0, 0); \ |
| dst += dst_stride; \ |
| __lsx_vstelm_d(dst0, dst, 0, 0); \ |
| dst += dst_stride; \ |
| __lsx_vstelm_d(dst0, dst, 0, 0); \ |
| dst += dst_stride; \ |
| __lsx_vstelm_d(dst0, dst, 0, 0); \ |
| dst += dst_stride; \ |
| __lsx_vstelm_d(dst0, dst, 0, 0); \ |
| dst += dst_stride; \ |
| __lsx_vstelm_d(dst0, dst, 0, 0); \ |
| } |
| |
| INTRA_DC_TL_8X8(top); |
| INTRA_DC_TL_8X8(left); |
| |
| void ff_dc_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, |
| const uint8_t *src_left, const uint8_t *src_top) |
| { |
| __m128i tmp0, tmp1, dst0; |
| ptrdiff_t stride2 = dst_stride << 1; |
| ptrdiff_t stride3 = stride2 + dst_stride; |
| ptrdiff_t stride4 = stride2 << 1; |
| |
| tmp0 = __lsx_vld(src_top, 0); |
| tmp1 = __lsx_vld(src_left, 0); |
| DUP2_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp0, tmp1); |
| dst0 = __lsx_vadd_h(tmp0, tmp1); |
| dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); |
| dst0 = __lsx_vhaddw_du_wu(dst0, dst0); |
| dst0 = __lsx_vhaddw_qu_du(dst0, dst0); |
| dst0 = __lsx_vsrari_w(dst0, 5); |
| dst0 = __lsx_vreplvei_b(dst0, 0); |
| LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, |
| dst_stride, stride2, stride3, stride4); |
| dst += stride4; |
| LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, |
| dst_stride, stride2, stride3, stride4); |
| } |
| |
| #define INTRA_DC_TL_16X16(dir) \ |
| void ff_dc_##dir##_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ |
| const uint8_t *left, \ |
| const uint8_t *top) \ |
| { \ |
| __m128i tmp0, dst0; \ |
| ptrdiff_t stride2 = dst_stride << 1; \ |
| ptrdiff_t stride3 = stride2 + dst_stride; \ |
| ptrdiff_t stride4 = stride2 << 1; \ |
| \ |
| tmp0 = __lsx_vld(dir, 0); \ |
| dst0 = __lsx_vhaddw_hu_bu(tmp0, tmp0); \ |
| dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); \ |
| dst0 = __lsx_vhaddw_du_wu(dst0, dst0); \ |
| dst0 = __lsx_vhaddw_qu_du(dst0, dst0); \ |
| dst0 = __lsx_vsrari_w(dst0, 4); \ |
| dst0 = __lsx_vreplvei_b(dst0, 0); \ |
| LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, \ |
| dst_stride, stride2, stride3, stride4); \ |
| dst += stride4; \ |
| LSX_ST_8(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst, \ |
| dst_stride, stride2, stride3, stride4); \ |
| } |
| |
| INTRA_DC_TL_16X16(top); |
| INTRA_DC_TL_16X16(left); |
| |
| void ff_dc_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, |
| const uint8_t *src_left, const uint8_t *src_top) |
| { |
| __m128i tmp0, tmp1, tmp2, tmp3, dst0; |
| |
| DUP2_ARG2(__lsx_vld, src_top, 0, src_top, 16, tmp0, tmp1); |
| DUP2_ARG2(__lsx_vld, src_left, 0, src_left, 16, tmp2, tmp3); |
| DUP4_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp2, tmp2, |
| tmp3, tmp3, tmp0, tmp1, tmp2, tmp3); |
| DUP2_ARG2(__lsx_vadd_h, tmp0, tmp1, tmp2, tmp3, tmp0, tmp1); |
| dst0 = __lsx_vadd_h(tmp0, tmp1); |
| dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); |
| dst0 = __lsx_vhaddw_du_wu(dst0, dst0); |
| dst0 = __lsx_vhaddw_qu_du(dst0, dst0); |
| dst0 = __lsx_vsrari_w(dst0, 6); |
| dst0 = __lsx_vreplvei_b(dst0, 0); |
| LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, |
| dst, dst_stride); |
| LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, |
| dst, dst_stride); |
| LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, |
| dst, dst_stride); |
| LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, |
| dst, dst_stride); |
| } |
| |
| #define INTRA_DC_TL_32X32(dir) \ |
| void ff_dc_##dir##_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ |
| const uint8_t *left, \ |
| const uint8_t *top) \ |
| { \ |
| __m128i tmp0, tmp1, dst0; \ |
| \ |
| DUP2_ARG2(__lsx_vld, dir, 0, dir, 16, tmp0, tmp1); \ |
| DUP2_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp0, tmp1); \ |
| dst0 = __lsx_vadd_h(tmp0, tmp1); \ |
| dst0 = __lsx_vhaddw_wu_hu(dst0, dst0); \ |
| dst0 = __lsx_vhaddw_du_wu(dst0, dst0); \ |
| dst0 = __lsx_vhaddw_qu_du(dst0, dst0); \ |
| dst0 = __lsx_vsrari_w(dst0, 5); \ |
| dst0 = __lsx_vreplvei_b(dst0, 0); \ |
| LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, \ |
| dst, dst_stride); \ |
| LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, \ |
| dst, dst_stride); \ |
| LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, \ |
| dst, dst_stride); \ |
| LSX_ST_8X16(dst0, dst0, dst0, dst0, dst0, dst0, dst0, dst0, \ |
| dst, dst_stride); \ |
| } |
| |
| INTRA_DC_TL_32X32(top); |
| INTRA_DC_TL_32X32(left); |
| |
| #define INTRA_PREDICT_VALDC_16X16_LSX(val) \ |
| void ff_dc_##val##_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ |
| const uint8_t *left, const uint8_t *top) \ |
| { \ |
| __m128i out = __lsx_vldi(val); \ |
| ptrdiff_t stride2 = dst_stride << 1; \ |
| ptrdiff_t stride3 = stride2 + dst_stride; \ |
| ptrdiff_t stride4 = stride2 << 1; \ |
| \ |
| LSX_ST_8(out, out, out, out, out, out, out, out, dst, \ |
| dst_stride, stride2, stride3, stride4); \ |
| dst += stride4; \ |
| LSX_ST_8(out, out, out, out, out, out, out, out, dst, \ |
| dst_stride, stride2, stride3, stride4); \ |
| } |
| |
| INTRA_PREDICT_VALDC_16X16_LSX(127); |
| INTRA_PREDICT_VALDC_16X16_LSX(128); |
| INTRA_PREDICT_VALDC_16X16_LSX(129); |
| |
| #define INTRA_PREDICT_VALDC_32X32_LSX(val) \ |
| void ff_dc_##val##_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, \ |
| const uint8_t *left, const uint8_t *top) \ |
| { \ |
| __m128i out = __lsx_vldi(val); \ |
| \ |
| LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\ |
| LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\ |
| LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\ |
| LSX_ST_8X16(out, out, out, out, out, out, out, out, dst, dst_stride);\ |
| } |
| |
| INTRA_PREDICT_VALDC_32X32_LSX(127); |
| INTRA_PREDICT_VALDC_32X32_LSX(128); |
| INTRA_PREDICT_VALDC_32X32_LSX(129); |
| |
| void ff_tm_4x4_lsx(uint8_t *dst, ptrdiff_t dst_stride, |
| const uint8_t *src_left, const uint8_t *src_top_ptr) |
| { |
| uint8_t top_left = src_top_ptr[-1]; |
| __m128i tmp0, tmp1, tmp2, tmp3, reg0, reg1; |
| __m128i src0, src1, src2, src3; |
| __m128i dst0, dst1, dst2, dst3; |
| |
| reg0 = __lsx_vreplgr2vr_h(top_left); |
| reg1 = __lsx_vld(src_top_ptr, 0); |
| DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left, |
| 3, tmp3, tmp2, tmp1, tmp0); |
| DUP4_ARG2(__lsx_vilvl_b, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vhaddw_hu_bu, src0, src0, src1, src1, src2, src2, src3, |
| src3, dst0, dst1, dst2, dst3); |
| DUP4_ARG2(__lsx_vssub_hu, dst0, reg0, dst1, reg0, dst2, reg0, dst3, reg0, |
| dst0, dst1, dst2, dst3); |
| DUP4_ARG2(__lsx_vsat_hu, dst0, 7, dst1, 7, dst2, 7, dst3, 7, |
| dst0, dst1, dst2, dst3); |
| DUP2_ARG2(__lsx_vpickev_b, dst1, dst0, dst3, dst2, dst0, dst1); |
| __lsx_vstelm_w(dst0, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_w(dst0, dst, 0, 2); |
| dst += dst_stride; |
| __lsx_vstelm_w(dst1, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_w(dst1, dst, 0, 2); |
| } |
| |
| void ff_tm_8x8_lsx(uint8_t *dst, ptrdiff_t dst_stride, |
| const uint8_t *src_left, const uint8_t *src_top_ptr) |
| { |
| uint8_t top_left = src_top_ptr[-1]; |
| __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; |
| __m128i src0, src1, src2, src3, src4, src5, src6, src7; |
| __m128i reg0, reg1; |
| |
| reg0 = __lsx_vreplgr2vr_h(top_left); |
| reg1 = __lsx_vld(src_top_ptr, 0); |
| DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left, |
| 3, tmp7, tmp6, tmp5, tmp4); |
| DUP4_ARG2(__lsx_vldrepl_b, src_left, 4, src_left, 5, src_left, 6, src_left, |
| 7, tmp3, tmp2, tmp1, tmp0); |
| DUP4_ARG2(__lsx_vilvl_b, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, reg1, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vilvl_b, tmp4, reg1, tmp5, reg1, tmp6, reg1, tmp7, reg1, |
| src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vhaddw_hu_bu, src0, src0, src1, src1, src2, src2, src3, |
| src3, src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vhaddw_hu_bu, src4, src4, src5, src5, src6, src6, src7, |
| src7, src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, |
| src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, |
| src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vpickev_b, src1, src0, src3, src2, src5, src4, src7, src6, |
| src0, src1, src2, src3); |
| __lsx_vstelm_d(src0, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_d(src0, dst, 0, 1); |
| dst += dst_stride; |
| __lsx_vstelm_d(src1, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_d(src1, dst, 0, 1); |
| dst += dst_stride; |
| __lsx_vstelm_d(src2, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_d(src2, dst, 0, 1); |
| dst += dst_stride; |
| __lsx_vstelm_d(src3, dst, 0, 0); |
| dst += dst_stride; |
| __lsx_vstelm_d(src3, dst, 0, 1); |
| } |
| |
| void ff_tm_16x16_lsx(uint8_t *dst, ptrdiff_t dst_stride, |
| const uint8_t *src_left, const uint8_t *src_top_ptr) |
| { |
| uint8_t top_left = src_top_ptr[-1]; |
| __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; |
| __m128i tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15; |
| __m128i src0, src1, src2, src3, src4, src5, src6, src7; |
| __m128i reg0, reg1; |
| ptrdiff_t stride2 = dst_stride << 1; |
| ptrdiff_t stride3 = stride2 + dst_stride; |
| ptrdiff_t stride4 = stride2 << 1; |
| |
| reg0 = __lsx_vreplgr2vr_h(top_left); |
| reg1 = __lsx_vld(src_top_ptr, 0); |
| DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, src_left, |
| 3, tmp15, tmp14, tmp13, tmp12); |
| DUP4_ARG2(__lsx_vldrepl_b, src_left, 4, src_left, 5, src_left, 6, src_left, |
| 7, tmp11, tmp10, tmp9, tmp8); |
| DUP4_ARG2(__lsx_vldrepl_b, src_left, 8, src_left, 9, src_left, 10, |
| src_left, 11, tmp7, tmp6, tmp5, tmp4); |
| DUP4_ARG2(__lsx_vldrepl_b, src_left, 12, src_left, 13, src_left, 14, |
| src_left, 15, tmp3, tmp2, tmp1, tmp0); |
| DUP4_ARG2(__lsx_vaddwev_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, |
| reg1, src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vaddwod_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, tmp3, |
| reg1, src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, |
| src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, |
| src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, |
| tmp0, tmp1, tmp2, tmp3); |
| DUP4_ARG2(__lsx_vaddwev_h_bu, tmp4, reg1, tmp5, reg1, tmp6, reg1, tmp7, |
| reg1, src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vaddwod_h_bu, tmp4, reg1, tmp5, reg1, tmp6, reg1, tmp7, |
| reg1, src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, |
| src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, |
| src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, |
| tmp4, tmp5, tmp6, tmp7); |
| DUP4_ARG2(__lsx_vaddwev_h_bu, tmp8, reg1, tmp9, reg1, tmp10, reg1, tmp11, |
| reg1, src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vaddwod_h_bu, tmp8, reg1, tmp9, reg1, tmp10, reg1, tmp11, |
| reg1, src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, |
| src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, |
| src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, |
| tmp8, tmp9, tmp10, tmp11); |
| DUP4_ARG2(__lsx_vaddwev_h_bu, tmp12, reg1, tmp13, reg1, tmp14, reg1, |
| tmp15, reg1, src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vaddwod_h_bu, tmp12, reg1, tmp13, reg1, tmp14, reg1, |
| tmp15, reg1, src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, reg0, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, reg0, |
| src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, |
| src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, src3, |
| tmp12, tmp13, tmp14, tmp15); |
| LSX_ST_8(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, dst, |
| dst_stride, stride2, stride3, stride4); |
| dst += stride4; |
| LSX_ST_8(tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15, dst, |
| dst_stride, stride2, stride3, stride4); |
| } |
| |
| void ff_tm_32x32_lsx(uint8_t *dst, ptrdiff_t dst_stride, |
| const uint8_t *src_left, const uint8_t *src_top_ptr) |
| { |
| uint8_t top_left = src_top_ptr[-1]; |
| uint32_t loop_cnt; |
| __m128i tmp0, tmp1, tmp2, tmp3, reg0, reg1, reg2; |
| __m128i src0, src1, src2, src3, src4, src5, src6, src7; |
| __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; |
| |
| reg0 = __lsx_vreplgr2vr_h(top_left); |
| DUP2_ARG2(__lsx_vld, src_top_ptr, 0, src_top_ptr, 16, reg1, reg2); |
| |
| src_left += 28; |
| for (loop_cnt = 8; loop_cnt--;) { |
| DUP4_ARG2(__lsx_vldrepl_b, src_left, 0, src_left, 1, src_left, 2, |
| src_left, 3, tmp3, tmp2, tmp1, tmp0); |
| src_left -= 4; |
| DUP4_ARG2(__lsx_vaddwev_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, |
| tmp3, reg1, src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vaddwod_h_bu, tmp0, reg1, tmp1, reg1, tmp2, reg1, |
| tmp3, reg1, src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vssub_hu, src0, reg0, src1, reg0, src2, reg0, src3, |
| reg0, src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vssub_hu, src4, reg0, src5, reg0, src6, reg0, src7, |
| reg0, src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vaddwev_h_bu, tmp0, reg2, tmp1, reg2, tmp2, reg2, |
| tmp3, reg2, dst0, dst1, dst2, dst3); |
| DUP4_ARG2(__lsx_vaddwod_h_bu, tmp0, reg2, tmp1, reg2, tmp2, reg2, |
| tmp3, reg2, dst4, dst5, dst6, dst7); |
| DUP4_ARG2(__lsx_vssub_hu, dst0, reg0, dst1, reg0, dst2, reg0, dst3, |
| reg0, dst0, dst1, dst2, dst3); |
| DUP4_ARG2(__lsx_vssub_hu, dst4, reg0, dst5, reg0, dst6, reg0, dst7, |
| reg0, dst4, dst5, dst6, dst7); |
| DUP4_ARG2(__lsx_vsat_hu, src0, 7, src1, 7, src2, 7, src3, 7, |
| src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vsat_hu, src4, 7, src5, 7, src6, 7, src7, 7, |
| src4, src5, src6, src7); |
| DUP4_ARG2(__lsx_vsat_hu, dst0, 7, dst1, 7, dst2, 7, dst3, 7, |
| dst0, dst1, dst2, dst3); |
| DUP4_ARG2(__lsx_vsat_hu, dst4, 7, dst5, 7, dst6, 7, dst7, 7, |
| dst4, dst5, dst6, dst7); |
| DUP4_ARG2(__lsx_vpackev_b, src4, src0, src5, src1, src6, src2, src7, |
| src3, src0, src1, src2, src3); |
| DUP4_ARG2(__lsx_vpackev_b, dst4, dst0, dst5, dst1, dst6, dst2, dst7, |
| dst3, dst0, dst1, dst2, dst3); |
| __lsx_vst(src0, dst, 0); |
| __lsx_vst(dst0, dst, 16); |
| dst += dst_stride; |
| __lsx_vst(src1, dst, 0); |
| __lsx_vst(dst1, dst, 16); |
| dst += dst_stride; |
| __lsx_vst(src2, dst, 0); |
| __lsx_vst(dst2, dst, 16); |
| dst += dst_stride; |
| __lsx_vst(src3, dst, 0); |
| __lsx_vst(dst3, dst, 16); |
| dst += dst_stride; |
| } |
| } |