| /* | |
| * Copyright 2026 The LibYuv Project Authors. All rights reserved. | |
| * | |
| * Use of this source code is governed by a BSD-style license | |
| * that can be found in the LICENSE file in the root of the source | |
| * tree. An additional intellectual property rights grant can be found | |
| * in the file PATENTS. All contributing project authors may | |
| * be found in the AUTHORS file in the root of the source tree. | |
| */ | |
| #include "libyuv/row.h" | |
| #include "libyuv/scale_row.h" | |
| #if !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) | |
| #include <wasm_simd128.h> | |
| #ifdef __cplusplus | |
| namespace libyuv { | |
| extern "C" { | |
| #endif | |
| // Arbitrary i8x16.shuffle masks are rematerialized on every use by V8, so | |
| // shuffles that do not map to a single instruction use swizzle with masks | |
| // loaded before the loop. Mask indices of 128 produce 0. | |
| #ifdef __wasm_relaxed_simd__ | |
| #define LIBYUV_WASM_SWIZZLE wasm_i8x16_relaxed_swizzle | |
| #else | |
| #define LIBYUV_WASM_SWIZZLE wasm_i8x16_swizzle | |
| #endif | |
| // Load a constant vector. The empty asm hides the address so the compiler | |
| // keeps the vector in a local instead of sinking a v128.const into the loop. | |
| static inline v128_t LoadConst(const uint8_t* p) { | |
| asm("" : "+r"(p)); | |
| return wasm_v128_load(p); | |
| } | |
| #ifdef HAS_SCALEROWDOWN2_WASMSIMD | |
| // Point sample 32 pixels to 16, keeping the odd pixels. | |
| void ScaleRowDown2_WASMSIMD(const uint8_t* src_ptr, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_ptr, | |
| int dst_width) { | |
| (void)src_stride; | |
| while (dst_width > 0) { | |
| v128_t a = wasm_u16x8_shr(wasm_v128_load(src_ptr), 8); | |
| v128_t b = wasm_u16x8_shr(wasm_v128_load(src_ptr + 16), 8); | |
| wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(a, b)); | |
| src_ptr += 32; | |
| dst_ptr += 16; | |
| dst_width -= 16; | |
| } | |
| } | |
| // Average pairs of pixels: (s0 + s1 + 1) >> 1. | |
| void ScaleRowDown2Linear_WASMSIMD(const uint8_t* src_ptr, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_ptr, | |
| int dst_width) { | |
| const v128_t zero = wasm_i16x8_const_splat(0); | |
| (void)src_stride; | |
| while (dst_width > 0) { | |
| v128_t a = wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_ptr)); | |
| v128_t b = wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_ptr + 16)); | |
| a = wasm_u16x8_avgr(a, zero); | |
| b = wasm_u16x8_avgr(b, zero); | |
| wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(a, b)); | |
| src_ptr += 32; | |
| dst_ptr += 16; | |
| dst_width -= 16; | |
| } | |
| } | |
| // Average 2x2 boxes: (s0 + s1 + t0 + t1 + 2) >> 2. | |
| void ScaleRowDown2Box_WASMSIMD(const uint8_t* src_ptr, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_ptr, | |
| int dst_width) { | |
| const uint8_t* t = src_ptr + src_stride; | |
| const v128_t two = wasm_i16x8_const_splat(2); | |
| while (dst_width > 0) { | |
| v128_t a = wasm_i16x8_add( | |
| wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_ptr)), | |
| wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(t))); | |
| v128_t b = wasm_i16x8_add( | |
| wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(src_ptr + 16)), | |
| wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(t + 16))); | |
| a = wasm_u16x8_shr(wasm_i16x8_add(a, two), 2); | |
| b = wasm_u16x8_shr(wasm_i16x8_add(b, two), 2); | |
| wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(a, b)); | |
| src_ptr += 32; | |
| t += 32; | |
| dst_ptr += 16; | |
| dst_width -= 16; | |
| } | |
| } | |
| #endif // HAS_SCALEROWDOWN2_WASMSIMD | |
| #ifdef HAS_SCALEROWDOWN4_WASMSIMD | |
| // Point sample 64 pixels to 16, keeping pixel 2 of every 4. Odd 16 bit | |
| // lanes then even bytes, which are unzips (uzp on Arm, pack on x86). | |
| void ScaleRowDown4_WASMSIMD(const uint8_t* src_ptr, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_ptr, | |
| int dst_width) { | |
| const v128_t mask = wasm_i16x8_const_splat(0xff); | |
| (void)src_stride; | |
| while (dst_width > 0) { | |
| v128_t a = wasm_i16x8_shuffle(wasm_v128_load(src_ptr), | |
| wasm_v128_load(src_ptr + 16), 1, 3, 5, 7, 9, | |
| 11, 13, 15); | |
| v128_t b = wasm_i16x8_shuffle(wasm_v128_load(src_ptr + 32), | |
| wasm_v128_load(src_ptr + 48), 1, 3, 5, 7, 9, | |
| 11, 13, 15); | |
| wasm_v128_store(dst_ptr, wasm_u8x16_narrow_i16x8(wasm_v128_and(a, mask), | |
| wasm_v128_and(b, mask))); | |
| src_ptr += 64; | |
| dst_ptr += 16; | |
| dst_width -= 16; | |
| } | |
| } | |
| // Average 4x4 boxes: (sum + 8) >> 4. 8 pixels per loop. | |
| void ScaleRowDown4Box_WASMSIMD(const uint8_t* src_ptr, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_ptr, | |
| int dst_width) { | |
| const v128_t eight = wasm_i16x8_const_splat(8); | |
| while (dst_width > 0) { | |
| v128_t a = wasm_i16x8_const_splat(0); | |
| v128_t b = wasm_i16x8_const_splat(0); | |
| const uint8_t* s = src_ptr; | |
| for (int i = 0; i < 4; ++i) { | |
| a = wasm_i16x8_add(a, | |
| wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(s))); | |
| b = wasm_i16x8_add( | |
| b, wasm_u16x8_extadd_pairwise_u8x16(wasm_v128_load(s + 16))); | |
| s += src_stride; | |
| } | |
| a = wasm_u16x8_narrow_i32x4(wasm_u32x4_extadd_pairwise_u16x8(a), | |
| wasm_u32x4_extadd_pairwise_u16x8(b)); | |
| a = wasm_u16x8_shr(wasm_i16x8_add(a, eight), 4); | |
| wasm_v128_store64_lane(dst_ptr, wasm_u8x16_narrow_i16x8(a, a), 0); | |
| src_ptr += 32; | |
| dst_ptr += 8; | |
| dst_width -= 8; | |
| } | |
| } | |
| #endif // HAS_SCALEROWDOWN4_WASMSIMD | |
| #ifdef HAS_SCALEROWDOWN34_WASMSIMD | |
| static const uint8_t kShuf34[8][16] = { | |
| // ScaleRowDown34: 16 bytes from a, then b, then 8 bytes from b. | |
| {0, 1, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 128, 128, 128, 128}, | |
| {128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 1, 3, 4}, | |
| {5, 7, 8, 9, 11, 12, 13, 15, 128, 128, 128, 128, 128, 128, 128, 128}, | |
| // Filter34: s0 s1 s3 and s0 s2 s2 of every 4. | |
| {0, 1, 3, 3, 4, 5, 7, 7, 8, 9, 11, 11, 12, 13, 15, 15}, | |
| {0, 2, 2, 3, 4, 6, 6, 7, 8, 10, 10, 11, 12, 14, 14, 15}, | |
| // Store34: bytes 0, 1, 2 of every 4 from a, then b, then 8 from b. | |
| {0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 128, 128, 128, 128}, | |
| {128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 0, 1, 2, 4}, | |
| {5, 6, 8, 9, 10, 12, 13, 14, 128, 128, 128, 128, 128, 128, 128, 128}, | |
| }; | |
| // Point sample 32 pixels to 24, keeping pixels 0, 1 and 3 of every 4. | |
| void ScaleRowDown34_WASMSIMD(const uint8_t* src_ptr, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_ptr, | |
| int dst_width) { | |
| const v128_t m0 = LoadConst(kShuf34[0]); | |
| const v128_t m1 = LoadConst(kShuf34[1]); | |
| const v128_t m2 = LoadConst(kShuf34[2]); | |
| (void)src_stride; | |
| while (dst_width > 0) { | |
| v128_t a = wasm_v128_load(src_ptr); | |
| v128_t b = wasm_v128_load(src_ptr + 16); | |
| wasm_v128_store(dst_ptr, wasm_v128_or(LIBYUV_WASM_SWIZZLE(a, m0), | |
| LIBYUV_WASM_SWIZZLE(b, m1))); | |
| wasm_v128_store64_lane(dst_ptr + 16, LIBYUV_WASM_SWIZZLE(b, m2), 0); | |
| src_ptr += 32; | |
| dst_ptr += 24; | |
| dst_width -= 24; | |
| } | |
| } | |
| // Rounding-down average: (a + b) >> 1. | |
| static inline v128_t AvgFloor(v128_t a, v128_t b) { | |
| return wasm_i8x16_sub( | |
| wasm_u8x16_avgr(a, b), | |
| wasm_v128_and(wasm_v128_xor(a, b), wasm_i8x16_const_splat(1))); | |
| } | |
| // Horizontal 4 to 3 filter of 16 pixels. Result bytes 4n+0..2 are | |
| // (3 * s0 + s1 + 2) >> 2, (s1 + s2 + 1) >> 1, (s2 + 3 * s3 + 2) >> 2 | |
| // using (3 * a + b + 2) >> 2 == avgr(a, (a + b) >> 1). Byte 4n+3 is unused. | |
| static inline v128_t Filter34(v128_t s, v128_t mp, v128_t ma) { | |
| v128_t p = LIBYUV_WASM_SWIZZLE(s, mp); // s0 s1 s3 | |
| v128_t a = LIBYUV_WASM_SWIZZLE(s, ma); // s0 s2 s2 | |
| v128_t b = wasm_u32x4_shr(s, 8); // s1 s2 s3 | |
| return wasm_u8x16_avgr(p, AvgFloor(a, b)); | |
| } | |
| // Filter rows 1 and 2 together, 1 : 1 | |
| void ScaleRowDown34_1_Box_WASMSIMD(const uint8_t* src_ptr, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_ptr, | |
| int dst_width) { | |
| const uint8_t* t = src_ptr + src_stride; | |
| const v128_t mp = LoadConst(kShuf34[3]); | |
| const v128_t ma = LoadConst(kShuf34[4]); | |
| const v128_t m0 = LoadConst(kShuf34[5]); | |
| const v128_t m1 = LoadConst(kShuf34[6]); | |
| const v128_t m2 = LoadConst(kShuf34[7]); | |
| while (dst_width > 0) { | |
| v128_t a = wasm_u8x16_avgr(Filter34(wasm_v128_load(src_ptr), mp, ma), | |
| Filter34(wasm_v128_load(t), mp, ma)); | |
| v128_t b = wasm_u8x16_avgr(Filter34(wasm_v128_load(src_ptr + 16), mp, ma), | |
| Filter34(wasm_v128_load(t + 16), mp, ma)); | |
| wasm_v128_store(dst_ptr, wasm_v128_or(LIBYUV_WASM_SWIZZLE(a, m0), | |
| LIBYUV_WASM_SWIZZLE(b, m1))); | |
| wasm_v128_store64_lane(dst_ptr + 16, LIBYUV_WASM_SWIZZLE(b, m2), 0); | |
| src_ptr += 32; | |
| t += 32; | |
| dst_ptr += 24; | |
| dst_width -= 24; | |
| } | |
| } | |
| // Filter rows 0 and 1 together, 3 : 1 | |
| void ScaleRowDown34_0_Box_WASMSIMD(const uint8_t* src_ptr, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_ptr, | |
| int dst_width) { | |
| const uint8_t* t = src_ptr + src_stride; | |
| const v128_t mp = LoadConst(kShuf34[3]); | |
| const v128_t ma = LoadConst(kShuf34[4]); | |
| const v128_t m0 = LoadConst(kShuf34[5]); | |
| const v128_t m1 = LoadConst(kShuf34[6]); | |
| const v128_t m2 = LoadConst(kShuf34[7]); | |
| while (dst_width > 0) { | |
| v128_t a0 = Filter34(wasm_v128_load(src_ptr), mp, ma); | |
| v128_t b0 = Filter34(wasm_v128_load(src_ptr + 16), mp, ma); | |
| v128_t a1 = Filter34(wasm_v128_load(t), mp, ma); | |
| v128_t b1 = Filter34(wasm_v128_load(t + 16), mp, ma); | |
| v128_t a = wasm_u8x16_avgr(a0, AvgFloor(a0, a1)); | |
| v128_t b = wasm_u8x16_avgr(b0, AvgFloor(b0, b1)); | |
| wasm_v128_store(dst_ptr, wasm_v128_or(LIBYUV_WASM_SWIZZLE(a, m0), | |
| LIBYUV_WASM_SWIZZLE(b, m1))); | |
| wasm_v128_store64_lane(dst_ptr + 16, LIBYUV_WASM_SWIZZLE(b, m2), 0); | |
| src_ptr += 32; | |
| t += 32; | |
| dst_ptr += 24; | |
| dst_width -= 24; | |
| } | |
| } | |
| #endif // HAS_SCALEROWDOWN34_WASMSIMD | |
| #ifdef HAS_SCALEROWDOWN38_WASMSIMD | |
| static const uint8_t kShuf38[3][16] = { | |
| // ScaleRowDown38: 6 bytes from a, then 6 bytes from b. | |
| {0, 3, 6, 8, 11, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}, | |
| {128, 128, 128, 128, 128, 128, 0, 3, 6, 8, 11, 14, 128, 128, 128, 128}, | |
| // Store38: bytes 0, 1, 3 of every 4. | |
| {0, 1, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 128, 128, 128, 128}, | |
| }; | |
| // Point sample 32 pixels to 12, keeping pixels 0, 3 and 6 of every 8. | |
| void ScaleRowDown38_WASMSIMD(const uint8_t* src_ptr, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_ptr, | |
| int dst_width) { | |
| const v128_t m0 = LoadConst(kShuf38[0]); | |
| const v128_t m1 = LoadConst(kShuf38[1]); | |
| (void)src_stride; | |
| while (dst_width > 0) { | |
| v128_t d = | |
| wasm_v128_or(LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_ptr), m0), | |
| LIBYUV_WASM_SWIZZLE(wasm_v128_load(src_ptr + 16), m1)); | |
| wasm_v128_store64_lane(dst_ptr, d, 0); | |
| wasm_v128_store32_lane(dst_ptr + 8, d, 2); | |
| src_ptr += 32; | |
| dst_ptr += 12; | |
| dst_width -= 12; | |
| } | |
| } | |
| // Sums of columns 0-2, 3-5 and 6-7 of 8 columns of u16 are scaled by the | |
| // i16 factors in lanes 0, 3 and 6 of k. Returns the high 16 bits of the | |
| // products in 32 bit lanes 0, 1 and 3. | |
| static inline v128_t Sum38(v128_t s, v128_t k) { | |
| const v128_t zero = wasm_i16x8_const_splat(0); | |
| v128_t s1 = wasm_i8x16_shuffle(s, zero, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, | |
| 13, 14, 15, 16, 17); | |
| v128_t s2 = wasm_i8x16_shuffle(s, zero, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, | |
| 15, 16, 17, 18, 19); | |
| s = wasm_i16x8_add(wasm_i16x8_add(s, s1), s2); | |
| return wasm_u32x4_shr(wasm_i32x4_dot_i16x8(s, k), 16); | |
| } | |
| // Pack 12 results from Sum38 and store them. | |
| static inline void Store38(uint8_t* dst_ptr, | |
| v128_t m, | |
| v128_t a, | |
| v128_t b, | |
| v128_t c, | |
| v128_t d) { | |
| v128_t r = wasm_u8x16_narrow_i16x8(wasm_u16x8_narrow_i32x4(a, b), | |
| wasm_u16x8_narrow_i32x4(c, d)); | |
| r = LIBYUV_WASM_SWIZZLE(r, m); | |
| wasm_v128_store64_lane(dst_ptr, r, 0); | |
| wasm_v128_store32_lane(dst_ptr + 8, r, 2); | |
| } | |
| // 8x3 -> 3x1 | |
| void ScaleRowDown38_3_Box_WASMSIMD(const uint8_t* src_ptr, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_ptr, | |
| int dst_width) { | |
| const v128_t k = | |
| wasm_i16x8_const(65536 / 9, 0, 0, 65536 / 9, 0, 0, 65536 / 6, 0); | |
| const v128_t m = LoadConst(kShuf38[2]); | |
| while (dst_width > 0) { | |
| v128_t s[4]; | |
| for (int i = 0; i < 4; ++i) { | |
| const uint8_t* p = src_ptr + i * 8; | |
| s[i] = wasm_i16x8_add(wasm_i16x8_add(wasm_u16x8_load8x8(p), | |
| wasm_u16x8_load8x8(p + src_stride)), | |
| wasm_u16x8_load8x8(p + src_stride * 2)); | |
| } | |
| Store38(dst_ptr, m, Sum38(s[0], k), Sum38(s[1], k), Sum38(s[2], k), | |
| Sum38(s[3], k)); | |
| src_ptr += 32; | |
| dst_ptr += 12; | |
| dst_width -= 12; | |
| } | |
| } | |
| // 8x2 -> 3x1 | |
| void ScaleRowDown38_2_Box_WASMSIMD(const uint8_t* src_ptr, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_ptr, | |
| int dst_width) { | |
| const v128_t k = | |
| wasm_i16x8_const(65536 / 6, 0, 0, 65536 / 6, 0, 0, 65536 / 4, 0); | |
| const v128_t m = LoadConst(kShuf38[2]); | |
| while (dst_width > 0) { | |
| v128_t s[4]; | |
| for (int i = 0; i < 4; ++i) { | |
| const uint8_t* p = src_ptr + i * 8; | |
| s[i] = wasm_i16x8_add(wasm_u16x8_load8x8(p), | |
| wasm_u16x8_load8x8(p + src_stride)); | |
| } | |
| Store38(dst_ptr, m, Sum38(s[0], k), Sum38(s[1], k), Sum38(s[2], k), | |
| Sum38(s[3], k)); | |
| src_ptr += 32; | |
| dst_ptr += 12; | |
| dst_width -= 12; | |
| } | |
| } | |
| #endif // HAS_SCALEROWDOWN38_WASMSIMD | |
| #ifdef HAS_SCALEFILTERCOLS_WASMSIMD | |
| // Bilinear column filtering with 7 bit fractions, matching ScaleFilterCols_C: | |
| // a + ((f * (b - a) + 64) >> 7). 8 pixels per loop. The fraction | |
| // (x >> 9) & 0x7f only uses the low 16 bits of x, so x is tracked in 16 bit | |
| // lanes. | |
| void ScaleFilterCols_WASMSIMD(uint8_t* dst_ptr, | |
| const uint8_t* src_ptr, | |
| int dst_width, | |
| int x, | |
| int dx) { | |
| uint32_t ux = (uint32_t)x; | |
| const uint32_t udx = (uint32_t)dx; | |
| v128_t xv = wasm_i16x8_make((int16_t)ux, (int16_t)(ux + udx), | |
| (int16_t)(ux + udx * 2), (int16_t)(ux + udx * 3), | |
| (int16_t)(ux + udx * 4), (int16_t)(ux + udx * 5), | |
| (int16_t)(ux + udx * 6), (int16_t)(ux + udx * 7)); | |
| const v128_t dx8 = wasm_i16x8_splat((int16_t)(udx * 8)); | |
| const v128_t maskff = wasm_i16x8_const_splat(0xff); | |
| const v128_t round = wasm_i16x8_const_splat(64); | |
| while (dst_width > 0) { | |
| v128_t p = wasm_i16x8_const_splat(0); | |
| p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 0); | |
| ux += udx; | |
| p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 1); | |
| ux += udx; | |
| p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 2); | |
| ux += udx; | |
| p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 3); | |
| ux += udx; | |
| p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 4); | |
| ux += udx; | |
| p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 5); | |
| ux += udx; | |
| p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 6); | |
| ux += udx; | |
| p = wasm_v128_load16_lane(src_ptr + ((int)ux >> 16), p, 7); | |
| ux += udx; | |
| v128_t f = wasm_u16x8_shr(xv, 9); | |
| xv = wasm_i16x8_add(xv, dx8); | |
| v128_t a = wasm_v128_and(p, maskff); | |
| v128_t d = wasm_i16x8_sub(wasm_u16x8_shr(p, 8), a); | |
| d = wasm_i16x8_shr(wasm_i16x8_add(wasm_i16x8_mul(d, f), round), 7); | |
| d = wasm_i16x8_add(d, a); | |
| wasm_v128_store64_lane(dst_ptr, wasm_u8x16_narrow_i16x8(d, d), 0); | |
| dst_ptr += 8; | |
| dst_width -= 8; | |
| } | |
| } | |
| #endif // HAS_SCALEFILTERCOLS_WASMSIMD | |
| #if defined(HAS_SCALEARGBROWDOWN2_WASMSIMD) || \ | |
| defined(HAS_SCALEARGBROWDOWNEVEN_WASMSIMD) | |
| // Average 2x2 boxes of ARGB pixels. s0 s1 hold 4 pairs of pixels from row | |
| // 0 and t0 t1 hold the same pairs from row 1. Returns 4 pixels. | |
| static inline v128_t BoxARGB(v128_t s0, v128_t s1, v128_t t0, v128_t t1) { | |
| v128_t se = wasm_i32x4_shuffle(s0, s1, 0, 2, 4, 6); | |
| v128_t so = wasm_i32x4_shuffle(s0, s1, 1, 3, 5, 7); | |
| v128_t te = wasm_i32x4_shuffle(t0, t1, 0, 2, 4, 6); | |
| v128_t to = wasm_i32x4_shuffle(t0, t1, 1, 3, 5, 7); | |
| // Interleave even and odd pixels so pairwise adds sum the same channel. | |
| v128_t a = wasm_i16x8_add( | |
| wasm_u16x8_extadd_pairwise_u8x16(wasm_i8x16_shuffle( | |
| se, so, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23)), | |
| wasm_u16x8_extadd_pairwise_u8x16(wasm_i8x16_shuffle( | |
| te, to, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23))); | |
| v128_t b = wasm_i16x8_add(wasm_u16x8_extadd_pairwise_u8x16(wasm_i8x16_shuffle( | |
| se, so, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, | |
| 13, 29, 14, 30, 15, 31)), | |
| wasm_u16x8_extadd_pairwise_u8x16(wasm_i8x16_shuffle( | |
| te, to, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, | |
| 13, 29, 14, 30, 15, 31))); | |
| a = wasm_u16x8_shr(wasm_i16x8_add(a, wasm_i16x8_const_splat(2)), 2); | |
| b = wasm_u16x8_shr(wasm_i16x8_add(b, wasm_i16x8_const_splat(2)), 2); | |
| return wasm_u8x16_narrow_i16x8(a, b); | |
| } | |
| #endif | |
| #ifdef HAS_SCALEARGBROWDOWN2_WASMSIMD | |
| // Point sample 8 ARGB pixels to 4, keeping the odd pixels. | |
| void ScaleARGBRowDown2_WASMSIMD(const uint8_t* src_argb, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_argb, | |
| int dst_width) { | |
| (void)src_stride; | |
| while (dst_width > 0) { | |
| v128_t a = wasm_v128_load(src_argb); | |
| v128_t b = wasm_v128_load(src_argb + 16); | |
| wasm_v128_store(dst_argb, wasm_i32x4_shuffle(a, b, 1, 3, 5, 7)); | |
| src_argb += 32; | |
| dst_argb += 16; | |
| dst_width -= 4; | |
| } | |
| } | |
| // Average pairs of ARGB pixels. | |
| void ScaleARGBRowDown2Linear_WASMSIMD(const uint8_t* src_argb, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_argb, | |
| int dst_width) { | |
| (void)src_stride; | |
| while (dst_width > 0) { | |
| v128_t a = wasm_v128_load(src_argb); | |
| v128_t b = wasm_v128_load(src_argb + 16); | |
| wasm_v128_store(dst_argb, | |
| wasm_u8x16_avgr(wasm_i32x4_shuffle(a, b, 0, 2, 4, 6), | |
| wasm_i32x4_shuffle(a, b, 1, 3, 5, 7))); | |
| src_argb += 32; | |
| dst_argb += 16; | |
| dst_width -= 4; | |
| } | |
| } | |
| // Average 2x2 boxes of ARGB pixels. | |
| void ScaleARGBRowDown2Box_WASMSIMD(const uint8_t* src_argb, | |
| ptrdiff_t src_stride, | |
| uint8_t* dst_argb, | |
| int dst_width) { | |
| const uint8_t* t = src_argb + src_stride; | |
| while (dst_width > 0) { | |
| wasm_v128_store( | |
| dst_argb, | |
| BoxARGB(wasm_v128_load(src_argb), wasm_v128_load(src_argb + 16), | |
| wasm_v128_load(t), wasm_v128_load(t + 16))); | |
| src_argb += 32; | |
| t += 32; | |
| dst_argb += 16; | |
| dst_width -= 4; | |
| } | |
| } | |
| #endif // HAS_SCALEARGBROWDOWN2_WASMSIMD | |
| #ifdef HAS_SCALEARGBROWDOWNEVEN_WASMSIMD | |
| // Point sample ARGB pixels with an even step. 4 pixels per loop. | |
| void ScaleARGBRowDownEven_WASMSIMD(const uint8_t* src_argb, | |
| ptrdiff_t src_stride, | |
| int src_stepx, | |
| uint8_t* dst_argb, | |
| int dst_width) { | |
| const ptrdiff_t step = (ptrdiff_t)src_stepx * 4; | |
| (void)src_stride; | |
| while (dst_width > 0) { | |
| v128_t v = wasm_v128_load32_zero(src_argb); | |
| v = wasm_v128_load32_lane(src_argb + step, v, 1); | |
| v = wasm_v128_load32_lane(src_argb + step * 2, v, 2); | |
| v = wasm_v128_load32_lane(src_argb + step * 3, v, 3); | |
| wasm_v128_store(dst_argb, v); | |
| src_argb += step * 4; | |
| dst_argb += 16; | |
| dst_width -= 4; | |
| } | |
| } | |
| // Average 2x2 boxes of ARGB pixels with an even step. 4 pixels per loop. | |
| void ScaleARGBRowDownEvenBox_WASMSIMD(const uint8_t* src_argb, | |
| ptrdiff_t src_stride, | |
| int src_stepx, | |
| uint8_t* dst_argb, | |
| int dst_width) { | |
| const ptrdiff_t step = (ptrdiff_t)src_stepx * 4; | |
| while (dst_width > 0) { | |
| const uint8_t* t = src_argb + src_stride; | |
| v128_t s0 = wasm_v128_load64_zero(src_argb); | |
| v128_t t0 = wasm_v128_load64_zero(t); | |
| s0 = wasm_v128_load64_lane(src_argb + step, s0, 1); | |
| t0 = wasm_v128_load64_lane(t + step, t0, 1); | |
| v128_t s1 = wasm_v128_load64_zero(src_argb + step * 2); | |
| v128_t t1 = wasm_v128_load64_zero(t + step * 2); | |
| s1 = wasm_v128_load64_lane(src_argb + step * 3, s1, 1); | |
| t1 = wasm_v128_load64_lane(t + step * 3, t1, 1); | |
| wasm_v128_store(dst_argb, BoxARGB(s0, s1, t0, t1)); | |
| src_argb += step * 4; | |
| dst_argb += 16; | |
| dst_width -= 4; | |
| } | |
| } | |
| #endif // HAS_SCALEARGBROWDOWNEVEN_WASMSIMD | |
| #ifdef HAS_SCALEARGBCOLS_WASMSIMD | |
| // Point sample ARGB columns. 4 pixels per loop. | |
| void ScaleARGBCols_WASMSIMD(uint8_t* dst_argb, | |
| const uint8_t* src_argb, | |
| int dst_width, | |
| int x, | |
| int dx) { | |
| uint32_t ux = (uint32_t)x; | |
| const uint32_t udx = (uint32_t)dx; | |
| while (dst_width > 0) { | |
| v128_t v = wasm_v128_load32_zero(src_argb + ((int)ux >> 16) * 4); | |
| ux += udx; | |
| v = wasm_v128_load32_lane(src_argb + ((int)ux >> 16) * 4, v, 1); | |
| ux += udx; | |
| v = wasm_v128_load32_lane(src_argb + ((int)ux >> 16) * 4, v, 2); | |
| ux += udx; | |
| v = wasm_v128_load32_lane(src_argb + ((int)ux >> 16) * 4, v, 3); | |
| ux += udx; | |
| wasm_v128_store(dst_argb, v); | |
| dst_argb += 16; | |
| dst_width -= 4; | |
| } | |
| } | |
| #endif // HAS_SCALEARGBCOLS_WASMSIMD | |
| #ifdef HAS_SCALEARGBFILTERCOLS_WASMSIMD | |
| // Bilinear ARGB column filtering with 7 bit fractions, matching | |
| // ScaleARGBFilterCols_C: (a * (127 - f) + b * f) >> 7. 4 pixels per loop. | |
| // The fraction (x >> 9) & 0x7f only uses the low 16 bits of x, so x is | |
| // tracked in 16 bit lanes, 4 per pixel to match the channels. | |
| void ScaleARGBFilterCols_WASMSIMD(uint8_t* dst_argb, | |
| const uint8_t* src_argb, | |
| int dst_width, | |
| int x, | |
| int dx) { | |
| uint32_t ux = (uint32_t)x; | |
| const uint32_t udx = (uint32_t)dx; | |
| int16_t x0 = (int16_t)ux; | |
| int16_t x1 = (int16_t)(ux + udx); | |
| int16_t x2 = (int16_t)(ux + udx * 2); | |
| int16_t x3 = (int16_t)(ux + udx * 3); | |
| v128_t x01 = wasm_i16x8_make(x0, x0, x0, x0, x1, x1, x1, x1); | |
| v128_t x23 = wasm_i16x8_make(x2, x2, x2, x2, x3, x3, x3, x3); | |
| const v128_t dx4 = wasm_i16x8_splat((int16_t)(udx * 4)); | |
| const v128_t k7f = wasm_i16x8_const_splat(0x7f); | |
| while (dst_width > 0) { | |
| // Load pixel pairs (a, b) for 4 pixels. | |
| v128_t p0 = wasm_v128_load64_zero(src_argb + ((int)ux >> 16) * 4); | |
| ux += udx; | |
| p0 = wasm_v128_load64_lane(src_argb + ((int)ux >> 16) * 4, p0, 1); | |
| ux += udx; | |
| v128_t p1 = wasm_v128_load64_zero(src_argb + ((int)ux >> 16) * 4); | |
| ux += udx; | |
| p1 = wasm_v128_load64_lane(src_argb + ((int)ux >> 16) * 4, p1, 1); | |
| ux += udx; | |
| v128_t a = wasm_i32x4_shuffle(p0, p1, 0, 2, 4, 6); // a0 a1 a2 a3 | |
| v128_t b = wasm_i32x4_shuffle(p0, p1, 1, 3, 5, 7); // b0 b1 b2 b3 | |
| v128_t f01 = wasm_u16x8_shr(x01, 9); | |
| v128_t f23 = wasm_u16x8_shr(x23, 9); | |
| x01 = wasm_i16x8_add(x01, dx4); | |
| x23 = wasm_i16x8_add(x23, dx4); | |
| v128_t r0 = wasm_i16x8_add( | |
| wasm_i16x8_mul(wasm_u16x8_extend_low_u8x16(a), wasm_v128_xor(f01, k7f)), | |
| wasm_i16x8_mul(wasm_u16x8_extend_low_u8x16(b), f01)); | |
| v128_t r1 = | |
| wasm_i16x8_add(wasm_i16x8_mul(wasm_u16x8_extend_high_u8x16(a), | |
| wasm_v128_xor(f23, k7f)), | |
| wasm_i16x8_mul(wasm_u16x8_extend_high_u8x16(b), f23)); | |
| wasm_v128_store(dst_argb, wasm_u8x16_narrow_i16x8(wasm_u16x8_shr(r0, 7), | |
| wasm_u16x8_shr(r1, 7))); | |
| dst_argb += 16; | |
| dst_width -= 4; | |
| } | |
| } | |
| #endif // HAS_SCALEARGBFILTERCOLS_WASMSIMD | |
| #ifdef HAS_SCALEARGBCOLSUP2_WASMSIMD | |
| // Point sample ARGB up by 2x. 8 pixels per loop. | |
| void ScaleARGBColsUp2_WASMSIMD(uint8_t* dst_argb, | |
| const uint8_t* src_argb, | |
| int dst_width, | |
| int x, | |
| int dx) { | |
| (void)x; | |
| (void)dx; | |
| while (dst_width > 0) { | |
| // V8 lowers an Arm zip of a vector with itself to several instructions, | |
| // so zip with copies of the pixels loaded by load64_splat instead. | |
| v128_t s = wasm_v128_load(src_argb); | |
| v128_t t = wasm_v128_load64_splat(src_argb); | |
| v128_t u = wasm_v128_load64_splat(src_argb + 8); | |
| wasm_v128_store(dst_argb, wasm_i32x4_shuffle(s, t, 0, 4, 1, 5)); | |
| wasm_v128_store(dst_argb + 16, wasm_i32x4_shuffle(s, u, 2, 6, 3, 7)); | |
| src_argb += 16; | |
| dst_argb += 32; | |
| dst_width -= 8; | |
| } | |
| } | |
| #endif // HAS_SCALEARGBCOLSUP2_WASMSIMD | |
| #ifdef HAS_SCALEADDROW_WASMSIMD | |
| // Reads 16 bytes and accumulates to 16 shorts at a time. | |
| void ScaleAddRow_WASMSIMD(const uint8_t* src_ptr, | |
| uint16_t* dst_ptr, | |
| int src_width) { | |
| while (src_width > 0) { | |
| v128_t s = wasm_v128_load(src_ptr); | |
| v128_t d0 = wasm_v128_load(dst_ptr); | |
| v128_t d1 = wasm_v128_load(dst_ptr + 8); | |
| d0 = wasm_i16x8_add(d0, wasm_u16x8_extend_low_u8x16(s)); | |
| d1 = wasm_i16x8_add(d1, wasm_u16x8_extend_high_u8x16(s)); | |
| wasm_v128_store(dst_ptr, d0); | |
| wasm_v128_store(dst_ptr + 8, d1); | |
| src_ptr += 16; | |
| dst_ptr += 16; | |
| src_width -= 16; | |
| } | |
| } | |
| #endif // HAS_SCALEADDROW_WASMSIMD | |
| #ifdef HAS_SCALEUVADDCOLS3_WASMSIMD | |
| // Reads 24 UV pixels (48 uint16_t) and writes 8 UV pixels (16 bytes). | |
| void ScaleUVAddCols3_WASMSIMD(int dst_width, | |
| int scaleval, | |
| const uint16_t* src_uv, | |
| uint8_t* dst_uv) { | |
| const v128_t scale = wasm_i16x8_splat((int16_t)scaleval); | |
| while (dst_width > 0) { | |
| v128_t s0 = wasm_v128_load(src_uv); | |
| v128_t s1 = wasm_v128_load(src_uv + 8); | |
| v128_t s2 = wasm_v128_load(src_uv + 16); | |
| v128_t s3 = wasm_v128_load(src_uv + 24); | |
| v128_t s4 = wasm_v128_load(src_uv + 32); | |
| v128_t s5 = wasm_v128_load(src_uv + 40); | |
| src_uv += 48; | |
| // Group 0..3 of 3 UV pairs from s0, s1, s2 (dwords 0..11) | |
| v128_t a0 = wasm_i32x4_shuffle(s0, s1, 0, 3, 6, 1); | |
| v128_t b0 = wasm_i32x4_shuffle(s0, s1, 1, 4, 7, 2); | |
| v128_t c0 = wasm_i32x4_shuffle(s0, s1, 2, 5, 0, 3); | |
| a0 = wasm_i32x4_shuffle(a0, s2, 0, 1, 2, 5); | |
| b0 = wasm_i32x4_shuffle(b0, s2, 0, 1, 2, 6); | |
| c0 = wasm_i32x4_shuffle(c0, s2, 0, 1, 4, 7); | |
| v128_t sum0 = wasm_i16x8_add(wasm_i16x8_add(a0, b0), c0); | |
| // Group 4..7 of 3 UV pairs from s3, s4, s5 (dwords 0..11) | |
| v128_t a1 = wasm_i32x4_shuffle(s3, s4, 0, 3, 6, 1); | |
| v128_t b1 = wasm_i32x4_shuffle(s3, s4, 1, 4, 7, 2); | |
| v128_t c1 = wasm_i32x4_shuffle(s3, s4, 2, 5, 0, 3); | |
| a1 = wasm_i32x4_shuffle(a1, s5, 0, 1, 2, 5); | |
| b1 = wasm_i32x4_shuffle(b1, s5, 0, 1, 2, 6); | |
| c1 = wasm_i32x4_shuffle(c1, s5, 0, 1, 4, 7); | |
| v128_t sum1 = wasm_i16x8_add(wasm_i16x8_add(a1, b1), c1); | |
| // Multiply unsigned 16-bit sums by scaleval and shift right by 16. | |
| v128_t lo0 = wasm_u32x4_shr(wasm_u32x4_extmul_low_u16x8(sum0, scale), 16); | |
| v128_t hi0 = wasm_u32x4_shr(wasm_u32x4_extmul_high_u16x8(sum0, scale), 16); | |
| v128_t lo1 = wasm_u32x4_shr(wasm_u32x4_extmul_low_u16x8(sum1, scale), 16); | |
| v128_t hi1 = wasm_u32x4_shr(wasm_u32x4_extmul_high_u16x8(sum1, scale), 16); | |
| v128_t p0 = wasm_i16x8_narrow_i32x4(lo0, hi0); | |
| v128_t p1 = wasm_i16x8_narrow_i32x4(lo1, hi1); | |
| wasm_v128_store(dst_uv, wasm_u8x16_narrow_i16x8(p0, p1)); | |
| dst_uv += 16; | |
| dst_width -= 8; | |
| } | |
| } | |
| #endif // HAS_SCALEUVADDCOLS3_WASMSIMD | |
| #ifdef __cplusplus | |
| } // extern "C" | |
| } // namespace libyuv | |
| #endif | |
| #endif // !defined(LIBYUV_DISABLE_WASM) && defined(__wasm_simd128__) |