| /* |
| * Copyright 2013 The LibYuv Project Authors. All rights reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #include "libyuv/row.h" |
| #include "libyuv/scale_row.h" |
| |
| #ifdef __cplusplus |
| namespace libyuv { |
| extern "C" { |
| #endif |
| |
| // This module is for 32 bit Visual C x86 |
| #if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && \ |
| !defined(__clang__) && defined(_M_IX86) |
| |
| // Offsets for source bytes 0 to 9 |
| static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9, |
| 128, 128, 128, 128, 128, 128, 128, 128}; |
| |
| // Offsets for source bytes 11 to 20 with 8 subtracted = 3 to 12. |
| static const uvec8 kShuf1 = {3, 4, 5, 7, 8, 9, 11, 12, |
| 128, 128, 128, 128, 128, 128, 128, 128}; |
| |
| // Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31. |
| static const uvec8 kShuf2 = {5, 7, 8, 9, 11, 12, 13, 15, |
| 128, 128, 128, 128, 128, 128, 128, 128}; |
| |
| // Offsets for source bytes 0 to 10 |
| static const uvec8 kShuf01 = {0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10}; |
| |
| // Offsets for source bytes 10 to 21 with 8 subtracted = 3 to 13. |
| static const uvec8 kShuf11 = {2, 3, 4, 5, 5, 6, 6, 7, |
| 8, 9, 9, 10, 10, 11, 12, 13}; |
| |
| // Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31. |
| static const uvec8 kShuf21 = {5, 6, 6, 7, 8, 9, 9, 10, |
| 10, 11, 12, 13, 13, 14, 14, 15}; |
| |
| // Coefficients for source bytes 0 to 10 |
| static const uvec8 kMadd01 = {3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2}; |
| |
| // Coefficients for source bytes 10 to 21 |
| static const uvec8 kMadd11 = {1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1}; |
| |
| // Coefficients for source bytes 21 to 31 |
| static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3}; |
| |
| // Coefficients for source bytes 21 to 31 |
| static const vec16 kRound34 = {2, 2, 2, 2, 2, 2, 2, 2}; |
| |
| static const uvec8 kShuf38a = {0, 3, 6, 8, 11, 14, 128, 128, |
| 128, 128, 128, 128, 128, 128, 128, 128}; |
| |
| static const uvec8 kShuf38b = {128, 128, 128, 128, 128, 128, 0, 3, |
| 6, 8, 11, 14, 128, 128, 128, 128}; |
| |
| // Arrange words 0,3,6 into 0,1,2 |
| static const uvec8 kShufAc = {0, 1, 6, 7, 12, 13, 128, 128, |
| 128, 128, 128, 128, 128, 128, 128, 128}; |
| |
| // Arrange words 0,3,6 into 3,4,5 |
| static const uvec8 kShufAc3 = {128, 128, 128, 128, 128, 128, 0, 1, |
| 6, 7, 12, 13, 128, 128, 128, 128}; |
| |
| // Scaling values for boxes of 3x3 and 2x3 |
| static const uvec16 kScaleAc33 = {65536 / 9, 65536 / 9, 65536 / 6, 65536 / 9, |
| 65536 / 9, 65536 / 6, 0, 0}; |
| |
| // Arrange first value for pixels 0,1,2,3,4,5 |
| static const uvec8 kShufAb0 = {0, 128, 3, 128, 6, 128, 8, 128, |
| 11, 128, 14, 128, 128, 128, 128, 128}; |
| |
| // Arrange second value for pixels 0,1,2,3,4,5 |
| static const uvec8 kShufAb1 = {1, 128, 4, 128, 7, 128, 9, 128, |
| 12, 128, 15, 128, 128, 128, 128, 128}; |
| |
| // Arrange third value for pixels 0,1,2,3,4,5 |
| static const uvec8 kShufAb2 = {2, 128, 5, 128, 128, 128, 10, 128, |
| 13, 128, 128, 128, 128, 128, 128, 128}; |
| |
| // Scaling values for boxes of 3x2 and 2x2 |
| static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3, |
| 65536 / 3, 65536 / 2, 0, 0}; |
| |
| // Reads 32 pixels, throws half away and writes 16 pixels. |
| __declspec(naked) void ScaleRowDown2_SSSE3(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| mov eax, [esp + 4] // src_ptr |
| // src_stride ignored |
| mov edx, [esp + 12] // dst_ptr |
| mov ecx, [esp + 16] // dst_width |
| |
| wloop: |
| movdqu xmm0, [eax] |
| movdqu xmm1, [eax + 16] |
| lea eax, [eax + 32] |
| psrlw xmm0, 8 // isolate odd pixels. |
| psrlw xmm1, 8 |
| packuswb xmm0, xmm1 |
| movdqu [edx], xmm0 |
| lea edx, [edx + 16] |
| sub ecx, 16 |
| jg wloop |
| |
| ret |
| } |
| } |
| |
| // Blends 32x1 rectangle to 16x1. |
| __declspec(naked) void ScaleRowDown2Linear_SSSE3(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| mov eax, [esp + 4] // src_ptr |
| // src_stride |
| mov edx, [esp + 12] // dst_ptr |
| mov ecx, [esp + 16] // dst_width |
| |
| pcmpeqb xmm4, xmm4 // constant 0x0101 |
| psrlw xmm4, 15 |
| packuswb xmm4, xmm4 |
| pxor xmm5, xmm5 // constant 0 |
| |
| wloop: |
| movdqu xmm0, [eax] |
| movdqu xmm1, [eax + 16] |
| lea eax, [eax + 32] |
| pmaddubsw xmm0, xmm4 // horizontal add |
| pmaddubsw xmm1, xmm4 |
| pavgw xmm0, xmm5 // (x + 1) / 2 |
| pavgw xmm1, xmm5 |
| packuswb xmm0, xmm1 |
| movdqu [edx], xmm0 |
| lea edx, [edx + 16] |
| sub ecx, 16 |
| jg wloop |
| |
| ret |
| } |
| } |
| |
| // Blends 32x2 rectangle to 16x1. |
| __declspec(naked) void ScaleRowDown2Box_SSSE3(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| push esi |
| mov eax, [esp + 4 + 4] // src_ptr |
| mov esi, [esp + 4 + 8] // src_stride |
| mov edx, [esp + 4 + 12] // dst_ptr |
| mov ecx, [esp + 4 + 16] // dst_width |
| |
| pcmpeqb xmm4, xmm4 // constant 0x0101 |
| psrlw xmm4, 15 |
| packuswb xmm4, xmm4 |
| pxor xmm5, xmm5 // constant 0 |
| |
| wloop: |
| movdqu xmm0, [eax] |
| movdqu xmm1, [eax + 16] |
| movdqu xmm2, [eax + esi] |
| movdqu xmm3, [eax + esi + 16] |
| lea eax, [eax + 32] |
| pmaddubsw xmm0, xmm4 // horizontal add |
| pmaddubsw xmm1, xmm4 |
| pmaddubsw xmm2, xmm4 |
| pmaddubsw xmm3, xmm4 |
| paddw xmm0, xmm2 // vertical add |
| paddw xmm1, xmm3 |
| psrlw xmm0, 1 |
| psrlw xmm1, 1 |
| pavgw xmm0, xmm5 // (x + 1) / 2 |
| pavgw xmm1, xmm5 |
| packuswb xmm0, xmm1 |
| movdqu [edx], xmm0 |
| lea edx, [edx + 16] |
| sub ecx, 16 |
| jg wloop |
| |
| pop esi |
| ret |
| } |
| } |
| |
| #ifdef HAS_SCALEROWDOWN2_AVX2 |
| // Reads 64 pixels, throws half away and writes 32 pixels. |
| __declspec(naked) void ScaleRowDown2_AVX2(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| mov eax, [esp + 4] // src_ptr |
| // src_stride ignored |
| mov edx, [esp + 12] // dst_ptr |
| mov ecx, [esp + 16] // dst_width |
| |
| wloop: |
| vmovdqu ymm0, [eax] |
| vmovdqu ymm1, [eax + 32] |
| lea eax, [eax + 64] |
| vpsrlw ymm0, ymm0, 8 // isolate odd pixels. |
| vpsrlw ymm1, ymm1, 8 |
| vpackuswb ymm0, ymm0, ymm1 |
| vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb |
| vmovdqu [edx], ymm0 |
| lea edx, [edx + 32] |
| sub ecx, 32 |
| jg wloop |
| |
| vzeroupper |
| ret |
| } |
| } |
| |
| // Blends 64x1 rectangle to 32x1. |
| __declspec(naked) void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| mov eax, [esp + 4] // src_ptr |
| // src_stride |
| mov edx, [esp + 12] // dst_ptr |
| mov ecx, [esp + 16] // dst_width |
| |
| vpcmpeqb ymm4, ymm4, ymm4 // '1' constant, 8b |
| vpsrlw ymm4, ymm4, 15 |
| vpackuswb ymm4, ymm4, ymm4 |
| vpxor ymm5, ymm5, ymm5 // constant 0 |
| |
| wloop: |
| vmovdqu ymm0, [eax] |
| vmovdqu ymm1, [eax + 32] |
| lea eax, [eax + 64] |
| vpmaddubsw ymm0, ymm0, ymm4 // horizontal add |
| vpmaddubsw ymm1, ymm1, ymm4 |
| vpavgw ymm0, ymm0, ymm5 // (x + 1) / 2 |
| vpavgw ymm1, ymm1, ymm5 |
| vpackuswb ymm0, ymm0, ymm1 |
| vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb |
| vmovdqu [edx], ymm0 |
| lea edx, [edx + 32] |
| sub ecx, 32 |
| jg wloop |
| |
| vzeroupper |
| ret |
| } |
| } |
| |
| // For rounding, average = (sum + 2) / 4 |
| // becomes average((sum >> 1), 0) |
| // Blends 64x2 rectangle to 32x1. |
| __declspec(naked) void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| push esi |
| mov eax, [esp + 4 + 4] // src_ptr |
| mov esi, [esp + 4 + 8] // src_stride |
| mov edx, [esp + 4 + 12] // dst_ptr |
| mov ecx, [esp + 4 + 16] // dst_width |
| |
| vpcmpeqb ymm4, ymm4, ymm4 // '1' constant, 8b |
| vpsrlw ymm4, ymm4, 15 |
| vpackuswb ymm4, ymm4, ymm4 |
| vpxor ymm5, ymm5, ymm5 // constant 0 |
| |
| wloop: |
| vmovdqu ymm0, [eax] |
| vmovdqu ymm1, [eax + 32] |
| vmovdqu ymm2, [eax + esi] |
| vmovdqu ymm3, [eax + esi + 32] |
| lea eax, [eax + 64] |
| vpmaddubsw ymm0, ymm0, ymm4 // horizontal add |
| vpmaddubsw ymm1, ymm1, ymm4 |
| vpmaddubsw ymm2, ymm2, ymm4 |
| vpmaddubsw ymm3, ymm3, ymm4 |
| vpaddw ymm0, ymm0, ymm2 // vertical add |
| vpaddw ymm1, ymm1, ymm3 |
| vpsrlw ymm0, ymm0, 1 // (x + 2) / 4 = (x / 2 + 1) / 2 |
| vpsrlw ymm1, ymm1, 1 |
| vpavgw ymm0, ymm0, ymm5 // (x + 1) / 2 |
| vpavgw ymm1, ymm1, ymm5 |
| vpackuswb ymm0, ymm0, ymm1 |
| vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb |
| vmovdqu [edx], ymm0 |
| lea edx, [edx + 32] |
| sub ecx, 32 |
| jg wloop |
| |
| pop esi |
| vzeroupper |
| ret |
| } |
| } |
| #endif // HAS_SCALEROWDOWN2_AVX2 |
| |
| // Point samples 32 pixels to 8 pixels. |
| __declspec(naked) void ScaleRowDown4_SSSE3(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| mov eax, [esp + 4] // src_ptr |
| // src_stride ignored |
| mov edx, [esp + 12] // dst_ptr |
| mov ecx, [esp + 16] // dst_width |
| pcmpeqb xmm5, xmm5 // generate mask 0x00ff0000 |
| psrld xmm5, 24 |
| pslld xmm5, 16 |
| |
| wloop: |
| movdqu xmm0, [eax] |
| movdqu xmm1, [eax + 16] |
| lea eax, [eax + 32] |
| pand xmm0, xmm5 |
| pand xmm1, xmm5 |
| packuswb xmm0, xmm1 |
| psrlw xmm0, 8 |
| packuswb xmm0, xmm0 |
| movq qword ptr [edx], xmm0 |
| lea edx, [edx + 8] |
| sub ecx, 8 |
| jg wloop |
| |
| ret |
| } |
| } |
| |
| // Blends 32x4 rectangle to 8x1. |
| __declspec(naked) void ScaleRowDown4Box_SSSE3(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| push esi |
| push edi |
| mov eax, [esp + 8 + 4] // src_ptr |
| mov esi, [esp + 8 + 8] // src_stride |
| mov edx, [esp + 8 + 12] // dst_ptr |
| mov ecx, [esp + 8 + 16] // dst_width |
| lea edi, [esi + esi * 2] // src_stride * 3 |
| pcmpeqb xmm4, xmm4 // constant 0x0101 |
| psrlw xmm4, 15 |
| movdqa xmm5, xmm4 |
| packuswb xmm4, xmm4 |
| psllw xmm5, 3 // constant 0x0008 |
| |
| wloop: |
| movdqu xmm0, [eax] // average rows |
| movdqu xmm1, [eax + 16] |
| movdqu xmm2, [eax + esi] |
| movdqu xmm3, [eax + esi + 16] |
| pmaddubsw xmm0, xmm4 // horizontal add |
| pmaddubsw xmm1, xmm4 |
| pmaddubsw xmm2, xmm4 |
| pmaddubsw xmm3, xmm4 |
| paddw xmm0, xmm2 // vertical add rows 0, 1 |
| paddw xmm1, xmm3 |
| movdqu xmm2, [eax + esi * 2] |
| movdqu xmm3, [eax + esi * 2 + 16] |
| pmaddubsw xmm2, xmm4 |
| pmaddubsw xmm3, xmm4 |
| paddw xmm0, xmm2 // add row 2 |
| paddw xmm1, xmm3 |
| movdqu xmm2, [eax + edi] |
| movdqu xmm3, [eax + edi + 16] |
| lea eax, [eax + 32] |
| pmaddubsw xmm2, xmm4 |
| pmaddubsw xmm3, xmm4 |
| paddw xmm0, xmm2 // add row 3 |
| paddw xmm1, xmm3 |
| phaddw xmm0, xmm1 |
| paddw xmm0, xmm5 // + 8 for round |
| psrlw xmm0, 4 // /16 for average of 4 * 4 |
| packuswb xmm0, xmm0 |
| movq qword ptr [edx], xmm0 |
| lea edx, [edx + 8] |
| sub ecx, 8 |
| jg wloop |
| |
| pop edi |
| pop esi |
| ret |
| } |
| } |
| |
| #ifdef HAS_SCALEROWDOWN4_AVX2 |
| // Point samples 64 pixels to 16 pixels. |
| __declspec(naked) void ScaleRowDown4_AVX2(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| mov eax, [esp + 4] // src_ptr |
| // src_stride ignored |
| mov edx, [esp + 12] // dst_ptr |
| mov ecx, [esp + 16] // dst_width |
| vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff0000 |
| vpsrld ymm5, ymm5, 24 |
| vpslld ymm5, ymm5, 16 |
| |
| wloop: |
| vmovdqu ymm0, [eax] |
| vmovdqu ymm1, [eax + 32] |
| lea eax, [eax + 64] |
| vpand ymm0, ymm0, ymm5 |
| vpand ymm1, ymm1, ymm5 |
| vpackuswb ymm0, ymm0, ymm1 |
| vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb |
| vpsrlw ymm0, ymm0, 8 |
| vpackuswb ymm0, ymm0, ymm0 |
| vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb |
| vmovdqu [edx], xmm0 |
| lea edx, [edx + 16] |
| sub ecx, 16 |
| jg wloop |
| |
| vzeroupper |
| ret |
| } |
| } |
| |
| // Blends 64x4 rectangle to 16x1. |
| __declspec(naked) void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| push esi |
| push edi |
| mov eax, [esp + 8 + 4] // src_ptr |
| mov esi, [esp + 8 + 8] // src_stride |
| mov edx, [esp + 8 + 12] // dst_ptr |
| mov ecx, [esp + 8 + 16] // dst_width |
| lea edi, [esi + esi * 2] // src_stride * 3 |
| vpcmpeqb ymm4, ymm4, ymm4 // constant 0x0101 |
| vpsrlw ymm4, ymm4, 15 |
| vpsllw ymm5, ymm4, 3 // constant 0x0008 |
| vpackuswb ymm4, ymm4, ymm4 |
| |
| wloop: |
| vmovdqu ymm0, [eax] // average rows |
| vmovdqu ymm1, [eax + 32] |
| vmovdqu ymm2, [eax + esi] |
| vmovdqu ymm3, [eax + esi + 32] |
| vpmaddubsw ymm0, ymm0, ymm4 // horizontal add |
| vpmaddubsw ymm1, ymm1, ymm4 |
| vpmaddubsw ymm2, ymm2, ymm4 |
| vpmaddubsw ymm3, ymm3, ymm4 |
| vpaddw ymm0, ymm0, ymm2 // vertical add rows 0, 1 |
| vpaddw ymm1, ymm1, ymm3 |
| vmovdqu ymm2, [eax + esi * 2] |
| vmovdqu ymm3, [eax + esi * 2 + 32] |
| vpmaddubsw ymm2, ymm2, ymm4 |
| vpmaddubsw ymm3, ymm3, ymm4 |
| vpaddw ymm0, ymm0, ymm2 // add row 2 |
| vpaddw ymm1, ymm1, ymm3 |
| vmovdqu ymm2, [eax + edi] |
| vmovdqu ymm3, [eax + edi + 32] |
| lea eax, [eax + 64] |
| vpmaddubsw ymm2, ymm2, ymm4 |
| vpmaddubsw ymm3, ymm3, ymm4 |
| vpaddw ymm0, ymm0, ymm2 // add row 3 |
| vpaddw ymm1, ymm1, ymm3 |
| vphaddw ymm0, ymm0, ymm1 // mutates |
| vpermq ymm0, ymm0, 0xd8 // unmutate vphaddw |
| vpaddw ymm0, ymm0, ymm5 // + 8 for round |
| vpsrlw ymm0, ymm0, 4 // /32 for average of 4 * 4 |
| vpackuswb ymm0, ymm0, ymm0 |
| vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb |
| vmovdqu [edx], xmm0 |
| lea edx, [edx + 16] |
| sub ecx, 16 |
| jg wloop |
| |
| pop edi |
| pop esi |
| vzeroupper |
| ret |
| } |
| } |
| #endif // HAS_SCALEROWDOWN4_AVX2 |
| |
| // Point samples 32 pixels to 24 pixels. |
| // Produces three 8 byte values. For each 8 bytes, 16 bytes are read. |
| // Then shuffled to do the scaling. |
| |
| __declspec(naked) void ScaleRowDown34_SSSE3(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| mov eax, [esp + 4] // src_ptr |
| // src_stride ignored |
| mov edx, [esp + 12] // dst_ptr |
| mov ecx, [esp + 16] // dst_width |
| movdqa xmm3, xmmword ptr kShuf0 |
| movdqa xmm4, xmmword ptr kShuf1 |
| movdqa xmm5, xmmword ptr kShuf2 |
| |
| wloop: |
| movdqu xmm0, [eax] |
| movdqu xmm1, [eax + 16] |
| lea eax, [eax + 32] |
| movdqa xmm2, xmm1 |
| palignr xmm1, xmm0, 8 |
| pshufb xmm0, xmm3 |
| pshufb xmm1, xmm4 |
| pshufb xmm2, xmm5 |
| movq qword ptr [edx], xmm0 |
| movq qword ptr [edx + 8], xmm1 |
| movq qword ptr [edx + 16], xmm2 |
| lea edx, [edx + 24] |
| sub ecx, 24 |
| jg wloop |
| |
| ret |
| } |
| } |
| |
| // Blends 32x2 rectangle to 24x1 |
| // Produces three 8 byte values. For each 8 bytes, 16 bytes are read. |
| // Then shuffled to do the scaling. |
| |
| // Register usage: |
| // xmm0 src_row 0 |
| // xmm1 src_row 1 |
| // xmm2 shuf 0 |
| // xmm3 shuf 1 |
| // xmm4 shuf 2 |
| // xmm5 madd 0 |
| // xmm6 madd 1 |
| // xmm7 kRound34 |
| |
| // Note that movdqa+palign may be better than movdqu. |
| __declspec(naked) void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| push esi |
| mov eax, [esp + 4 + 4] // src_ptr |
| mov esi, [esp + 4 + 8] // src_stride |
| mov edx, [esp + 4 + 12] // dst_ptr |
| mov ecx, [esp + 4 + 16] // dst_width |
| movdqa xmm2, xmmword ptr kShuf01 |
| movdqa xmm3, xmmword ptr kShuf11 |
| movdqa xmm4, xmmword ptr kShuf21 |
| movdqa xmm5, xmmword ptr kMadd01 |
| movdqa xmm6, xmmword ptr kMadd11 |
| movdqa xmm7, xmmword ptr kRound34 |
| |
| wloop: |
| movdqu xmm0, [eax] // pixels 0..7 |
| movdqu xmm1, [eax + esi] |
| pavgb xmm0, xmm1 |
| pshufb xmm0, xmm2 |
| pmaddubsw xmm0, xmm5 |
| paddsw xmm0, xmm7 |
| psrlw xmm0, 2 |
| packuswb xmm0, xmm0 |
| movq qword ptr [edx], xmm0 |
| movdqu xmm0, [eax + 8] // pixels 8..15 |
| movdqu xmm1, [eax + esi + 8] |
| pavgb xmm0, xmm1 |
| pshufb xmm0, xmm3 |
| pmaddubsw xmm0, xmm6 |
| paddsw xmm0, xmm7 |
| psrlw xmm0, 2 |
| packuswb xmm0, xmm0 |
| movq qword ptr [edx + 8], xmm0 |
| movdqu xmm0, [eax + 16] // pixels 16..23 |
| movdqu xmm1, [eax + esi + 16] |
| lea eax, [eax + 32] |
| pavgb xmm0, xmm1 |
| pshufb xmm0, xmm4 |
| movdqa xmm1, xmmword ptr kMadd21 |
| pmaddubsw xmm0, xmm1 |
| paddsw xmm0, xmm7 |
| psrlw xmm0, 2 |
| packuswb xmm0, xmm0 |
| movq qword ptr [edx + 16], xmm0 |
| lea edx, [edx + 24] |
| sub ecx, 24 |
| jg wloop |
| |
| pop esi |
| ret |
| } |
| } |
| |
| // Note that movdqa+palign may be better than movdqu. |
| __declspec(naked) void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| push esi |
| mov eax, [esp + 4 + 4] // src_ptr |
| mov esi, [esp + 4 + 8] // src_stride |
| mov edx, [esp + 4 + 12] // dst_ptr |
| mov ecx, [esp + 4 + 16] // dst_width |
| movdqa xmm2, xmmword ptr kShuf01 |
| movdqa xmm3, xmmword ptr kShuf11 |
| movdqa xmm4, xmmword ptr kShuf21 |
| movdqa xmm5, xmmword ptr kMadd01 |
| movdqa xmm6, xmmword ptr kMadd11 |
| movdqa xmm7, xmmword ptr kRound34 |
| |
| wloop: |
| movdqu xmm0, [eax] // pixels 0..7 |
| movdqu xmm1, [eax + esi] |
| pavgb xmm1, xmm0 |
| pavgb xmm0, xmm1 |
| pshufb xmm0, xmm2 |
| pmaddubsw xmm0, xmm5 |
| paddsw xmm0, xmm7 |
| psrlw xmm0, 2 |
| packuswb xmm0, xmm0 |
| movq qword ptr [edx], xmm0 |
| movdqu xmm0, [eax + 8] // pixels 8..15 |
| movdqu xmm1, [eax + esi + 8] |
| pavgb xmm1, xmm0 |
| pavgb xmm0, xmm1 |
| pshufb xmm0, xmm3 |
| pmaddubsw xmm0, xmm6 |
| paddsw xmm0, xmm7 |
| psrlw xmm0, 2 |
| packuswb xmm0, xmm0 |
| movq qword ptr [edx + 8], xmm0 |
| movdqu xmm0, [eax + 16] // pixels 16..23 |
| movdqu xmm1, [eax + esi + 16] |
| lea eax, [eax + 32] |
| pavgb xmm1, xmm0 |
| pavgb xmm0, xmm1 |
| pshufb xmm0, xmm4 |
| movdqa xmm1, xmmword ptr kMadd21 |
| pmaddubsw xmm0, xmm1 |
| paddsw xmm0, xmm7 |
| psrlw xmm0, 2 |
| packuswb xmm0, xmm0 |
| movq qword ptr [edx + 16], xmm0 |
| lea edx, [edx+24] |
| sub ecx, 24 |
| jg wloop |
| |
| pop esi |
| ret |
| } |
| } |
| |
| // 3/8 point sampler |
| |
| // Scale 32 pixels to 12 |
| __declspec(naked) void ScaleRowDown38_SSSE3(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| mov eax, [esp + 4] // src_ptr |
| // src_stride ignored |
| mov edx, [esp + 12] // dst_ptr |
| mov ecx, [esp + 16] // dst_width |
| movdqa xmm4, xmmword ptr kShuf38a |
| movdqa xmm5, xmmword ptr kShuf38b |
| |
| xloop: |
| movdqu xmm0, [eax] // 16 pixels -> 0,1,2,3,4,5 |
| movdqu xmm1, [eax + 16] // 16 pixels -> 6,7,8,9,10,11 |
| lea eax, [eax + 32] |
| pshufb xmm0, xmm4 |
| pshufb xmm1, xmm5 |
| paddusb xmm0, xmm1 |
| |
| movq qword ptr [edx], xmm0 // write 12 pixels |
| movhlps xmm1, xmm0 |
| movd [edx + 8], xmm1 |
| lea edx, [edx + 12] |
| sub ecx, 12 |
| jg xloop |
| |
| ret |
| } |
| } |
| |
| // Scale 16x3 pixels to 6x1 with interpolation |
| __declspec(naked) void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| push esi |
| mov eax, [esp + 4 + 4] // src_ptr |
| mov esi, [esp + 4 + 8] // src_stride |
| mov edx, [esp + 4 + 12] // dst_ptr |
| mov ecx, [esp + 4 + 16] // dst_width |
| movdqa xmm2, xmmword ptr kShufAc |
| movdqa xmm3, xmmword ptr kShufAc3 |
| movdqa xmm4, xmmword ptr kScaleAc33 |
| pxor xmm5, xmm5 |
| |
| xloop: |
| movdqu xmm0, [eax] // sum up 3 rows into xmm0/1 |
| movdqu xmm6, [eax + esi] |
| movhlps xmm1, xmm0 |
| movhlps xmm7, xmm6 |
| punpcklbw xmm0, xmm5 |
| punpcklbw xmm1, xmm5 |
| punpcklbw xmm6, xmm5 |
| punpcklbw xmm7, xmm5 |
| paddusw xmm0, xmm6 |
| paddusw xmm1, xmm7 |
| movdqu xmm6, [eax + esi * 2] |
| lea eax, [eax + 16] |
| movhlps xmm7, xmm6 |
| punpcklbw xmm6, xmm5 |
| punpcklbw xmm7, xmm5 |
| paddusw xmm0, xmm6 |
| paddusw xmm1, xmm7 |
| |
| movdqa xmm6, xmm0 // 8 pixels -> 0,1,2 of xmm6 |
| psrldq xmm0, 2 |
| paddusw xmm6, xmm0 |
| psrldq xmm0, 2 |
| paddusw xmm6, xmm0 |
| pshufb xmm6, xmm2 |
| |
| movdqa xmm7, xmm1 // 8 pixels -> 3,4,5 of xmm6 |
| psrldq xmm1, 2 |
| paddusw xmm7, xmm1 |
| psrldq xmm1, 2 |
| paddusw xmm7, xmm1 |
| pshufb xmm7, xmm3 |
| paddusw xmm6, xmm7 |
| |
| pmulhuw xmm6, xmm4 // divide by 9,9,6, 9,9,6 |
| packuswb xmm6, xmm6 |
| |
| movd [edx], xmm6 // write 6 pixels |
| psrlq xmm6, 16 |
| movd [edx + 2], xmm6 |
| lea edx, [edx + 6] |
| sub ecx, 6 |
| jg xloop |
| |
| pop esi |
| ret |
| } |
| } |
| |
| // Scale 16x2 pixels to 6x1 with interpolation |
| __declspec(naked) void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| uint8_t* dst_ptr, |
| int dst_width) { |
| __asm { |
| push esi |
| mov eax, [esp + 4 + 4] // src_ptr |
| mov esi, [esp + 4 + 8] // src_stride |
| mov edx, [esp + 4 + 12] // dst_ptr |
| mov ecx, [esp + 4 + 16] // dst_width |
| movdqa xmm2, xmmword ptr kShufAb0 |
| movdqa xmm3, xmmword ptr kShufAb1 |
| movdqa xmm4, xmmword ptr kShufAb2 |
| movdqa xmm5, xmmword ptr kScaleAb2 |
| |
| xloop: |
| movdqu xmm0, [eax] // average 2 rows into xmm0 |
| movdqu xmm1, [eax + esi] |
| lea eax, [eax + 16] |
| pavgb xmm0, xmm1 |
| |
| movdqa xmm1, xmm0 // 16 pixels -> 0,1,2,3,4,5 of xmm1 |
| pshufb xmm1, xmm2 |
| movdqa xmm6, xmm0 |
| pshufb xmm6, xmm3 |
| paddusw xmm1, xmm6 |
| pshufb xmm0, xmm4 |
| paddusw xmm1, xmm0 |
| |
| pmulhuw xmm1, xmm5 // divide by 3,3,2, 3,3,2 |
| packuswb xmm1, xmm1 |
| |
| movd [edx], xmm1 // write 6 pixels |
| psrlq xmm1, 16 |
| movd [edx + 2], xmm1 |
| lea edx, [edx + 6] |
| sub ecx, 6 |
| jg xloop |
| |
| pop esi |
| ret |
| } |
| } |
| |
| // Reads 16 bytes and accumulates to 16 shorts at a time. |
| __declspec(naked) void ScaleAddRow_SSE2(const uint8_t* src_ptr, |
| uint16_t* dst_ptr, |
| int src_width) { |
| __asm { |
| mov eax, [esp + 4] // src_ptr |
| mov edx, [esp + 8] // dst_ptr |
| mov ecx, [esp + 12] // src_width |
| pxor xmm5, xmm5 |
| |
| // sum rows |
| xloop: |
| movdqu xmm3, [eax] // read 16 bytes |
| lea eax, [eax + 16] |
| movdqu xmm0, [edx] // read 16 words from destination |
| movdqu xmm1, [edx + 16] |
| movdqa xmm2, xmm3 |
| punpcklbw xmm2, xmm5 |
| punpckhbw xmm3, xmm5 |
| paddusw xmm0, xmm2 // sum 16 words |
| paddusw xmm1, xmm3 |
| movdqu [edx], xmm0 // write 16 words to destination |
| movdqu [edx + 16], xmm1 |
| lea edx, [edx + 32] |
| sub ecx, 16 |
| jg xloop |
| ret |
| } |
| } |
| |
| #ifdef HAS_SCALEADDROW_AVX2 |
| // Reads 32 bytes and accumulates to 32 shorts at a time. |
| __declspec(naked) void ScaleAddRow_AVX2(const uint8_t* src_ptr, |
| uint16_t* dst_ptr, |
| int src_width) { |
| __asm { |
| mov eax, [esp + 4] // src_ptr |
| mov edx, [esp + 8] // dst_ptr |
| mov ecx, [esp + 12] // src_width |
| vpxor ymm5, ymm5, ymm5 |
| |
| // sum rows |
| xloop: |
| vmovdqu ymm3, [eax] // read 32 bytes |
| lea eax, [eax + 32] |
| vpermq ymm3, ymm3, 0xd8 // unmutate for vpunpck |
| vpunpcklbw ymm2, ymm3, ymm5 |
| vpunpckhbw ymm3, ymm3, ymm5 |
| vpaddusw ymm0, ymm2, [edx] // sum 16 words |
| vpaddusw ymm1, ymm3, [edx + 32] |
| vmovdqu [edx], ymm0 // write 32 words to destination |
| vmovdqu [edx + 32], ymm1 |
| lea edx, [edx + 64] |
| sub ecx, 32 |
| jg xloop |
| |
| vzeroupper |
| ret |
| } |
| } |
| #endif // HAS_SCALEADDROW_AVX2 |
| |
| // Constant for making pixels signed to avoid pmaddubsw |
| // saturation. |
| static const uvec8 kFsub80 = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, |
| 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; |
| |
| // Constant for making pixels unsigned and adding .5 for rounding. |
| static const uvec16 kFadd40 = {0x4040, 0x4040, 0x4040, 0x4040, |
| 0x4040, 0x4040, 0x4040, 0x4040}; |
| |
| // Bilinear column filtering. SSSE3 version. |
| __declspec(naked) void ScaleFilterCols_SSSE3(uint8_t* dst_ptr, |
| const uint8_t* src_ptr, |
| int dst_width, |
| int x, |
| int dx) { |
| __asm { |
| push ebx |
| push esi |
| push edi |
| mov edi, [esp + 12 + 4] // dst_ptr |
| mov esi, [esp + 12 + 8] // src_ptr |
| mov ecx, [esp + 12 + 12] // dst_width |
| movd xmm2, [esp + 12 + 16] // x |
| movd xmm3, [esp + 12 + 20] // dx |
| mov eax, 0x04040000 // shuffle to line up fractions with pixel. |
| movd xmm5, eax |
| pcmpeqb xmm6, xmm6 // generate 0x007f for inverting fraction. |
| psrlw xmm6, 9 |
| pcmpeqb xmm7, xmm7 // generate 0x0001 |
| psrlw xmm7, 15 |
| pextrw eax, xmm2, 1 // get x0 integer. preroll |
| sub ecx, 2 |
| jl xloop29 |
| |
| movdqa xmm0, xmm2 // x1 = x0 + dx |
| paddd xmm0, xmm3 |
| punpckldq xmm2, xmm0 // x0 x1 |
| punpckldq xmm3, xmm3 // dx dx |
| paddd xmm3, xmm3 // dx * 2, dx * 2 |
| pextrw edx, xmm2, 3 // get x1 integer. preroll |
| |
| // 2 Pixel loop. |
| xloop2: |
| movdqa xmm1, xmm2 // x0, x1 fractions. |
| paddd xmm2, xmm3 // x += dx |
| movzx ebx, word ptr [esi + eax] // 2 source x0 pixels |
| movd xmm0, ebx |
| psrlw xmm1, 9 // 7 bit fractions. |
| movzx ebx, word ptr [esi + edx] // 2 source x1 pixels |
| movd xmm4, ebx |
| pshufb xmm1, xmm5 // 0011 |
| punpcklwd xmm0, xmm4 |
| psubb xmm0, xmmword ptr kFsub80 // make pixels signed. |
| pxor xmm1, xmm6 // 0..7f and 7f..0 |
| paddusb xmm1, xmm7 // +1 so 0..7f and 80..1 |
| pmaddubsw xmm1, xmm0 // 16 bit, 2 pixels. |
| pextrw eax, xmm2, 1 // get x0 integer. next iteration. |
| pextrw edx, xmm2, 3 // get x1 integer. next iteration. |
| paddw xmm1, xmmword ptr kFadd40 // make pixels unsigned and round. |
| psrlw xmm1, 7 // 8.7 fixed point to low 8 bits. |
| packuswb xmm1, xmm1 // 8 bits, 2 pixels. |
| movd ebx, xmm1 |
| mov [edi], bx |
| lea edi, [edi + 2] |
| sub ecx, 2 // 2 pixels |
| jge xloop2 |
| |
| xloop29: |
| add ecx, 2 - 1 |
| jl xloop99 |
| |
| // 1 pixel remainder |
| movzx ebx, word ptr [esi + eax] // 2 source x0 pixels |
| movd xmm0, ebx |
| psrlw xmm2, 9 // 7 bit fractions. |
| pshufb xmm2, xmm5 // 0011 |
| psubb xmm0, xmmword ptr kFsub80 // make pixels signed. |
| pxor xmm2, xmm6 // 0..7f and 7f..0 |
| paddusb xmm2, xmm7 // +1 so 0..7f and 80..1 |
| pmaddubsw xmm2, xmm0 // 16 bit |
| paddw xmm2, xmmword ptr kFadd40 // make pixels unsigned and round. |
| psrlw xmm2, 7 // 8.7 fixed point to low 8 bits. |
| packuswb xmm2, xmm2 // 8 bits |
| movd ebx, xmm2 |
| mov [edi], bl |
| |
| xloop99: |
| |
| pop edi |
| pop esi |
| pop ebx |
| ret |
| } |
| } |
| |
| // Reads 16 pixels, duplicates them and writes 32 pixels. |
| __declspec(naked) void ScaleColsUp2_SSE2(uint8_t* dst_ptr, |
| const uint8_t* src_ptr, |
| int dst_width, |
| int x, |
| int dx) { |
| __asm { |
| mov edx, [esp + 4] // dst_ptr |
| mov eax, [esp + 8] // src_ptr |
| mov ecx, [esp + 12] // dst_width |
| |
| wloop: |
| movdqu xmm0, [eax] |
| lea eax, [eax + 16] |
| movdqa xmm1, xmm0 |
| punpcklbw xmm0, xmm0 |
| punpckhbw xmm1, xmm1 |
| movdqu [edx], xmm0 |
| movdqu [edx + 16], xmm1 |
| lea edx, [edx + 32] |
| sub ecx, 32 |
| jg wloop |
| |
| ret |
| } |
| } |
| |
| // Reads 8 pixels, throws half away and writes 4 even pixels (0, 2, 4, 6) |
| __declspec(naked) void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb, |
| ptrdiff_t src_stride, |
| uint8_t* dst_argb, |
| int dst_width) { |
| __asm { |
| mov eax, [esp + 4] // src_argb |
| // src_stride ignored |
| mov edx, [esp + 12] // dst_argb |
| mov ecx, [esp + 16] // dst_width |
| |
| wloop: |
| movdqu xmm0, [eax] |
| movdqu xmm1, [eax + 16] |
| lea eax, [eax + 32] |
| shufps xmm0, xmm1, 0xdd |
| movdqu [edx], xmm0 |
| lea edx, [edx + 16] |
| sub ecx, 4 |
| jg wloop |
| |
| ret |
| } |
| } |
| |
| // Blends 8x1 rectangle to 4x1. |
| __declspec(naked) void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb, |
| ptrdiff_t src_stride, |
| uint8_t* dst_argb, |
| int dst_width) { |
| __asm { |
| mov eax, [esp + 4] // src_argb |
| // src_stride ignored |
| mov edx, [esp + 12] // dst_argb |
| mov ecx, [esp + 16] // dst_width |
| |
| wloop: |
| movdqu xmm0, [eax] |
| movdqu xmm1, [eax + 16] |
| lea eax, [eax + 32] |
| movdqa xmm2, xmm0 |
| shufps xmm0, xmm1, 0x88 // even pixels |
| shufps xmm2, xmm1, 0xdd // odd pixels |
| pavgb xmm0, xmm2 |
| movdqu [edx], xmm0 |
| lea edx, [edx + 16] |
| sub ecx, 4 |
| jg wloop |
| |
| ret |
| } |
| } |
| |
| // Blends 8x2 rectangle to 4x1. |
| __declspec(naked) void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb, |
| ptrdiff_t src_stride, |
| uint8_t* dst_argb, |
| int dst_width) { |
| __asm { |
| push esi |
| mov eax, [esp + 4 + 4] // src_argb |
| mov esi, [esp + 4 + 8] // src_stride |
| mov edx, [esp + 4 + 12] // dst_argb |
| mov ecx, [esp + 4 + 16] // dst_width |
| |
| wloop: |
| movdqu xmm0, [eax] |
| movdqu xmm1, [eax + 16] |
| movdqu xmm2, [eax + esi] |
| movdqu xmm3, [eax + esi + 16] |
| lea eax, [eax + 32] |
| pavgb xmm0, xmm2 // average rows |
| pavgb xmm1, xmm3 |
| movdqa xmm2, xmm0 // average columns (8 to 4 pixels) |
| shufps xmm0, xmm1, 0x88 // even pixels |
| shufps xmm2, xmm1, 0xdd // odd pixels |
| pavgb xmm0, xmm2 |
| movdqu [edx], xmm0 |
| lea edx, [edx + 16] |
| sub ecx, 4 |
| jg wloop |
| |
| pop esi |
| ret |
| } |
| } |
| |
| // Reads 4 pixels at a time. |
| __declspec(naked) void ScaleARGBRowDownEven_SSE2(const uint8_t* src_argb, |
| ptrdiff_t src_stride, |
| int src_stepx, |
| uint8_t* dst_argb, |
| int dst_width) { |
| __asm { |
| push ebx |
| push edi |
| mov eax, [esp + 8 + 4] // src_argb |
| // src_stride ignored |
| mov ebx, [esp + 8 + 12] // src_stepx |
| mov edx, [esp + 8 + 16] // dst_argb |
| mov ecx, [esp + 8 + 20] // dst_width |
| lea ebx, [ebx * 4] |
| lea edi, [ebx + ebx * 2] |
| |
| wloop: |
| movd xmm0, [eax] |
| movd xmm1, [eax + ebx] |
| punpckldq xmm0, xmm1 |
| movd xmm2, [eax + ebx * 2] |
| movd xmm3, [eax + edi] |
| lea eax, [eax + ebx * 4] |
| punpckldq xmm2, xmm3 |
| punpcklqdq xmm0, xmm2 |
| movdqu [edx], xmm0 |
| lea edx, [edx + 16] |
| sub ecx, 4 |
| jg wloop |
| |
| pop edi |
| pop ebx |
| ret |
| } |
| } |
| |
| // Blends four 2x2 to 4x1. |
| __declspec(naked) void ScaleARGBRowDownEvenBox_SSE2(const uint8_t* src_argb, |
| ptrdiff_t src_stride, |
| int src_stepx, |
| uint8_t* dst_argb, |
| int dst_width) { |
| __asm { |
| push ebx |
| push esi |
| push edi |
| mov eax, [esp + 12 + 4] // src_argb |
| mov esi, [esp + 12 + 8] // src_stride |
| mov ebx, [esp + 12 + 12] // src_stepx |
| mov edx, [esp + 12 + 16] // dst_argb |
| mov ecx, [esp + 12 + 20] // dst_width |
| lea esi, [eax + esi] // row1 pointer |
| lea ebx, [ebx * 4] |
| lea edi, [ebx + ebx * 2] |
| |
| wloop: |
| movq xmm0, qword ptr [eax] // row0 4 pairs |
| movhps xmm0, qword ptr [eax + ebx] |
| movq xmm1, qword ptr [eax + ebx * 2] |
| movhps xmm1, qword ptr [eax + edi] |
| lea eax, [eax + ebx * 4] |
| movq xmm2, qword ptr [esi] // row1 4 pairs |
| movhps xmm2, qword ptr [esi + ebx] |
| movq xmm3, qword ptr [esi + ebx * 2] |
| movhps xmm3, qword ptr [esi + edi] |
| lea esi, [esi + ebx * 4] |
| pavgb xmm0, xmm2 // average rows |
| pavgb xmm1, xmm3 |
| movdqa xmm2, xmm0 // average columns (8 to 4 pixels) |
| shufps xmm0, xmm1, 0x88 // even pixels |
| shufps xmm2, xmm1, 0xdd // odd pixels |
| pavgb xmm0, xmm2 |
| movdqu [edx], xmm0 |
| lea edx, [edx + 16] |
| sub ecx, 4 |
| jg wloop |
| |
| pop edi |
| pop esi |
| pop ebx |
| ret |
| } |
| } |
| |
| // Column scaling unfiltered. SSE2 version. |
| __declspec(naked) void ScaleARGBCols_SSE2(uint8_t* dst_argb, |
| const uint8_t* src_argb, |
| int dst_width, |
| int x, |
| int dx) { |
| __asm { |
| push edi |
| push esi |
| mov edi, [esp + 8 + 4] // dst_argb |
| mov esi, [esp + 8 + 8] // src_argb |
| mov ecx, [esp + 8 + 12] // dst_width |
| movd xmm2, [esp + 8 + 16] // x |
| movd xmm3, [esp + 8 + 20] // dx |
| |
| pshufd xmm2, xmm2, 0 // x0 x0 x0 x0 |
| pshufd xmm0, xmm3, 0x11 // dx 0 dx 0 |
| paddd xmm2, xmm0 |
| paddd xmm3, xmm3 // 0, 0, 0, dx * 2 |
| pshufd xmm0, xmm3, 0x05 // dx * 2, dx * 2, 0, 0 |
| paddd xmm2, xmm0 // x3 x2 x1 x0 |
| paddd xmm3, xmm3 // 0, 0, 0, dx * 4 |
| pshufd xmm3, xmm3, 0 // dx * 4, dx * 4, dx * 4, dx * 4 |
| |
| pextrw eax, xmm2, 1 // get x0 integer. |
| pextrw edx, xmm2, 3 // get x1 integer. |
| |
| cmp ecx, 0 |
| jle xloop99 |
| sub ecx, 4 |
| jl xloop49 |
| |
| // 4 Pixel loop. |
| xloop4: |
| movd xmm0, [esi + eax * 4] // 1 source x0 pixels |
| movd xmm1, [esi + edx * 4] // 1 source x1 pixels |
| pextrw eax, xmm2, 5 // get x2 integer. |
| pextrw edx, xmm2, 7 // get x3 integer. |
| paddd xmm2, xmm3 // x += dx |
| punpckldq xmm0, xmm1 // x0 x1 |
| |
| movd xmm1, [esi + eax * 4] // 1 source x2 pixels |
| movd xmm4, [esi + edx * 4] // 1 source x3 pixels |
| pextrw eax, xmm2, 1 // get x0 integer. next iteration. |
| pextrw edx, xmm2, 3 // get x1 integer. next iteration. |
| punpckldq xmm1, xmm4 // x2 x3 |
| punpcklqdq xmm0, xmm1 // x0 x1 x2 x3 |
| movdqu [edi], xmm0 |
| lea edi, [edi + 16] |
| sub ecx, 4 // 4 pixels |
| jge xloop4 |
| |
| xloop49: |
| test ecx, 2 |
| je xloop29 |
| |
| // 2 Pixels. |
| movd xmm0, [esi + eax * 4] // 1 source x0 pixels |
| movd xmm1, [esi + edx * 4] // 1 source x1 pixels |
| pextrw eax, xmm2, 5 // get x2 integer. |
| punpckldq xmm0, xmm1 // x0 x1 |
| |
| movq qword ptr [edi], xmm0 |
| lea edi, [edi + 8] |
| |
| xloop29: |
| test ecx, 1 |
| je xloop99 |
| |
| // 1 Pixels. |
| movd xmm0, [esi + eax * 4] // 1 source x2 pixels |
| movd dword ptr [edi], xmm0 |
| xloop99: |
| |
| pop esi |
| pop edi |
| ret |
| } |
| } |
| |
| // Bilinear row filtering combines 2x1 -> 1x1. SSSE3 version. |
| // TODO(fbarchard): Port to Neon |
| |
| // Shuffle table for arranging 2 pixels into pairs for pmaddubsw |
| static const uvec8 kShuffleColARGB = { |
| 0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u, // bbggrraa 1st pixel |
| 8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u // bbggrraa 2nd pixel |
| }; |
| |
| // Shuffle table for duplicating 2 fractions into 8 bytes each |
| static const uvec8 kShuffleFractions = { |
| 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, |
| }; |
| |
| __declspec(naked) void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb, |
| const uint8_t* src_argb, |
| int dst_width, |
| int x, |
| int dx) { |
| __asm { |
| push esi |
| push edi |
| mov edi, [esp + 8 + 4] // dst_argb |
| mov esi, [esp + 8 + 8] // src_argb |
| mov ecx, [esp + 8 + 12] // dst_width |
| movd xmm2, [esp + 8 + 16] // x |
| movd xmm3, [esp + 8 + 20] // dx |
| movdqa xmm4, xmmword ptr kShuffleColARGB |
| movdqa xmm5, xmmword ptr kShuffleFractions |
| pcmpeqb xmm6, xmm6 // generate 0x007f for inverting fraction. |
| psrlw xmm6, 9 |
| pextrw eax, xmm2, 1 // get x0 integer. preroll |
| sub ecx, 2 |
| jl xloop29 |
| |
| movdqa xmm0, xmm2 // x1 = x0 + dx |
| paddd xmm0, xmm3 |
| punpckldq xmm2, xmm0 // x0 x1 |
| punpckldq xmm3, xmm3 // dx dx |
| paddd xmm3, xmm3 // dx * 2, dx * 2 |
| pextrw edx, xmm2, 3 // get x1 integer. preroll |
| |
| // 2 Pixel loop. |
| xloop2: |
| movdqa xmm1, xmm2 // x0, x1 fractions. |
| paddd xmm2, xmm3 // x += dx |
| movq xmm0, qword ptr [esi + eax * 4] // 2 source x0 pixels |
| psrlw xmm1, 9 // 7 bit fractions. |
| movhps xmm0, qword ptr [esi + edx * 4] // 2 source x1 pixels |
| pshufb xmm1, xmm5 // 0000000011111111 |
| pshufb xmm0, xmm4 // arrange pixels into pairs |
| pxor xmm1, xmm6 // 0..7f and 7f..0 |
| pmaddubsw xmm0, xmm1 // argb_argb 16 bit, 2 pixels. |
| pextrw eax, xmm2, 1 // get x0 integer. next iteration. |
| pextrw edx, xmm2, 3 // get x1 integer. next iteration. |
| psrlw xmm0, 7 // argb 8.7 fixed point to low 8 bits. |
| packuswb xmm0, xmm0 // argb_argb 8 bits, 2 pixels. |
| movq qword ptr [edi], xmm0 |
| lea edi, [edi + 8] |
| sub ecx, 2 // 2 pixels |
| jge xloop2 |
| |
| xloop29: |
| |
| add ecx, 2 - 1 |
| jl xloop99 |
| |
| // 1 pixel remainder |
| psrlw xmm2, 9 // 7 bit fractions. |
| movq xmm0, qword ptr [esi + eax * 4] // 2 source x0 pixels |
| pshufb xmm2, xmm5 // 00000000 |
| pshufb xmm0, xmm4 // arrange pixels into pairs |
| pxor xmm2, xmm6 // 0..7f and 7f..0 |
| pmaddubsw xmm0, xmm2 // argb 16 bit, 1 pixel. |
| psrlw xmm0, 7 |
| packuswb xmm0, xmm0 // argb 8 bits, 1 pixel. |
| movd [edi], xmm0 |
| |
| xloop99: |
| |
| pop edi |
| pop esi |
| ret |
| } |
| } |
| |
| // Reads 4 pixels, duplicates them and writes 8 pixels. |
| __declspec(naked) void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb, |
| const uint8_t* src_argb, |
| int dst_width, |
| int x, |
| int dx) { |
| __asm { |
| mov edx, [esp + 4] // dst_argb |
| mov eax, [esp + 8] // src_argb |
| mov ecx, [esp + 12] // dst_width |
| |
| wloop: |
| movdqu xmm0, [eax] |
| lea eax, [eax + 16] |
| movdqa xmm1, xmm0 |
| punpckldq xmm0, xmm0 |
| punpckhdq xmm1, xmm1 |
| movdqu [edx], xmm0 |
| movdqu [edx + 16], xmm1 |
| lea edx, [edx + 32] |
| sub ecx, 8 |
| jg wloop |
| |
| ret |
| } |
| } |
| |
| // Divide num by div and return as 16.16 fixed point result. |
| __declspec(naked) int FixedDiv_X86(int num, int div) { |
| __asm { |
| mov eax, [esp + 4] // num |
| cdq // extend num to 64 bits |
| shld edx, eax, 16 // 32.16 |
| shl eax, 16 |
| idiv dword ptr [esp + 8] |
| ret |
| } |
| } |
| |
| // Divide num by div and return as 16.16 fixed point result. |
| __declspec(naked) int FixedDiv1_X86(int num, int div) { |
| __asm { |
| mov eax, [esp + 4] // num |
| mov ecx, [esp + 8] // denom |
| cdq // extend num to 64 bits |
| shld edx, eax, 16 // 32.16 |
| shl eax, 16 |
| sub eax, 0x00010001 |
| sbb edx, 0 |
| sub ecx, 1 |
| idiv ecx |
| ret |
| } |
| } |
| #endif // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) |
| |
| #ifdef __cplusplus |
| } // extern "C" |
| } // namespace libyuv |
| #endif |
