| /* |
| * Copyright 2024 The LibYuv Project Authors. All rights reserved. |
| * |
| * Use of this source code is governed by a BSD-style license |
| * that can be found in the LICENSE file in the root of the source |
| * tree. An additional intellectual property rights grant can be found |
| * in the file PATENTS. All contributing project authors may |
| * be found in the AUTHORS file in the root of the source tree. |
| */ |
| |
| #include "libyuv/row.h" |
| #include "libyuv/row_sve.h" |
| |
| #ifdef __cplusplus |
| namespace libyuv { |
| extern "C" { |
| #endif |
| |
| #if !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && \ |
| defined(__aarch64__) |
| |
| #define RGBTOARGB8_SVE_2X \ |
| /* Inputs: B: z16.h, G: z17.h, R: z18.h, A: z19.b */ \ |
| "uqshrnb z16.b, z16.h, #6 \n" /* B0 */ \ |
| "uqshrnb z17.b, z17.h, #6 \n" /* G0 */ \ |
| "uqshrnb z18.b, z18.h, #6 \n" /* R0 */ \ |
| "uqshrnt z16.b, z20.h, #6 \n" /* B1 */ \ |
| "uqshrnt z17.b, z21.h, #6 \n" /* G1 */ \ |
| "uqshrnt z18.b, z22.h, #6 \n" /* R1 */ |
| |
| __arm_locally_streaming void I444ToARGBRow_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_u, |
| const uint8_t* src_v, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| // Streaming-SVE only, no use of ZA tile. |
| uint64_t vl; |
| asm volatile( |
| "cntb %[vl] \n" |
| "ptrue p0.b \n" // |
| YUVTORGB_SVE_SETUP |
| "dup z19.b, #255 \n" // A |
| "subs %w[width], %w[width], %w[vl] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p1.b \n" |
| "1: \n" // |
| READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X |
| "subs %w[width], %w[width], %w[vl] \n" |
| "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" |
| "incb %[dst_argb], all, mul #4 \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| "whilelt p1.b, wzr, %w[width] \n" // |
| READYUV444_SVE_2X I444TORGB_SVE_2X RGBTOARGB8_SVE_2X |
| "st4b {z16.b, z17.b, z18.b, z19.b}, p1, [%[dst_argb]] \n" |
| |
| "99: \n" |
| : [src_y] "+r"(src_y), // %[src_y] |
| [src_u] "+r"(src_u), // %[src_u] |
| [src_v] "+r"(src_v), // %[src_v] |
| [dst_argb] "+r"(dst_argb), // %[dst_argb] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : [kUVCoeff] "r"(&yuvconstants->kUVCoeff), // %[kUVCoeff] |
| [kRGBCoeffBias] "r"(&yuvconstants->kRGBCoeffBias) // %[kRGBCoeffBias] |
| : "cc", "memory", YUVTORGB_SVE_REGS); |
| } |
| |
| __arm_locally_streaming void I444ToRGB24Row_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_u, |
| const uint8_t* src_v, |
| uint8_t* dst_rgb24, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I444ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_rgb24, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I400ToARGBRow_SME( |
| const uint8_t* src_y, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| // Streaming-SVE only, no use of ZA tile. |
| I400ToARGBRow_SVE_SC(src_y, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I422ToARGBRow_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_u, |
| const uint8_t* src_v, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| // Streaming-SVE only, no use of ZA tile. |
| I422ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I422ToRGB24Row_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_u, |
| const uint8_t* src_v, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I422ToRGB24Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I422ToRGB565Row_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_u, |
| const uint8_t* src_v, |
| uint8_t* dst_rgb565, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I422ToRGB565Row_SVE_SC(src_y, src_u, src_v, dst_rgb565, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I422ToARGB1555Row_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_u, |
| const uint8_t* src_v, |
| uint8_t* dst_argb1555, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I422ToARGB1555Row_SVE_SC(src_y, src_u, src_v, dst_argb1555, yuvconstants, |
| width); |
| } |
| |
| __arm_locally_streaming void I422ToARGB4444Row_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_u, |
| const uint8_t* src_v, |
| uint8_t* dst_argb4444, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I422ToARGB4444Row_SVE_SC(src_y, src_u, src_v, dst_argb4444, yuvconstants, |
| width); |
| } |
| |
| __arm_locally_streaming void I422ToRGBARow_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_u, |
| const uint8_t* src_v, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I422ToRGBARow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I422ToAR30Row_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_u, |
| const uint8_t* src_v, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I422ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I422AlphaToARGBRow_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_u, |
| const uint8_t* src_v, |
| const uint8_t* src_a, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I422AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, |
| width); |
| } |
| |
| __arm_locally_streaming void I444AlphaToARGBRow_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_u, |
| const uint8_t* src_v, |
| const uint8_t* src_a, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I444AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, |
| width); |
| } |
| |
| __arm_locally_streaming void NV12ToARGBRow_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_uv, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| NV12ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void NV21ToARGBRow_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_vu, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| NV21ToARGBRow_SVE_SC(src_y, src_vu, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void NV12ToRGB24Row_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_uv, |
| uint8_t* dst_rgb24, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| NV12ToRGB24Row_SVE_SC(src_y, src_uv, dst_rgb24, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void NV21ToRGB24Row_SME( |
| const uint8_t* src_y, |
| const uint8_t* src_vu, |
| uint8_t* dst_rgb24, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| NV21ToRGB24Row_SVE_SC(src_y, src_vu, dst_rgb24, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void YUY2ToARGBRow_SME( |
| const uint8_t* src_yuy2, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| YUY2ToARGBRow_SVE_SC(src_yuy2, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void UYVYToARGBRow_SME( |
| const uint8_t* src_uyvy, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| UYVYToARGBRow_SVE_SC(src_uyvy, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I210ToARGBRow_SME( |
| const uint16_t* src_y, |
| const uint16_t* src_u, |
| const uint16_t* src_v, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I210ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I210AlphaToARGBRow_SME( |
| const uint16_t* src_y, |
| const uint16_t* src_u, |
| const uint16_t* src_v, |
| const uint16_t* src_a, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I210AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, |
| width); |
| } |
| |
| __arm_locally_streaming void I210ToAR30Row_SME( |
| const uint16_t* src_y, |
| const uint16_t* src_u, |
| const uint16_t* src_v, |
| uint8_t* dst_ar30, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I210ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void P210ToARGBRow_SME( |
| const uint16_t* src_y, |
| const uint16_t* src_uv, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| P210ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void P210ToAR30Row_SME( |
| const uint16_t* src_y, |
| const uint16_t* src_uv, |
| uint8_t* dst_ar30, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| P210ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I410ToARGBRow_SME( |
| const uint16_t* src_y, |
| const uint16_t* src_u, |
| const uint16_t* src_v, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I410ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I410AlphaToARGBRow_SME( |
| const uint16_t* src_y, |
| const uint16_t* src_u, |
| const uint16_t* src_v, |
| const uint16_t* src_a, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I410AlphaToARGBRow_SVE_SC(src_y, src_u, src_v, src_a, dst_argb, yuvconstants, |
| width); |
| } |
| |
| __arm_locally_streaming void I410ToAR30Row_SME( |
| const uint16_t* src_y, |
| const uint16_t* src_u, |
| const uint16_t* src_v, |
| uint8_t* dst_ar30, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I410ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void P410ToARGBRow_SME( |
| const uint16_t* src_y, |
| const uint16_t* src_uv, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| P410ToARGBRow_SVE_SC(src_y, src_uv, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void P410ToAR30Row_SME( |
| const uint16_t* src_y, |
| const uint16_t* src_uv, |
| uint8_t* dst_ar30, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| P410ToAR30Row_SVE_SC(src_y, src_uv, dst_ar30, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I212ToAR30Row_SME( |
| const uint16_t* src_y, |
| const uint16_t* src_u, |
| const uint16_t* src_v, |
| uint8_t* dst_ar30, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I212ToAR30Row_SVE_SC(src_y, src_u, src_v, dst_ar30, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void I212ToARGBRow_SME( |
| const uint16_t* src_y, |
| const uint16_t* src_u, |
| const uint16_t* src_v, |
| uint8_t* dst_argb, |
| const struct YuvConstants* yuvconstants, |
| int width) { |
| I212ToARGBRow_SVE_SC(src_y, src_u, src_v, dst_argb, yuvconstants, width); |
| } |
| |
| __arm_locally_streaming void MultiplyRow_16_SME(const uint16_t* src_y, |
| uint16_t* dst_y, |
| int scale, |
| int width) { |
| // Streaming-SVE only, no use of ZA tile. |
| int vl; |
| asm volatile( |
| "cnth %x[vl] \n" |
| "mov z0.h, %w[scale] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p0.h \n" |
| "1: \n" |
| "ld1h {z1.h}, p0/z, [%[src_y]] \n" |
| "incb %[src_y] \n" |
| "mul z1.h, z0.h, z1.h \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "st1h {z1.h}, p0, [%[dst_y]] \n" |
| "incb %[dst_y] \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| "whilelt p0.h, wzr, %w[width] \n" |
| "ld1h {z1.h}, p0/z, [%[src_y]] \n" |
| "mul z1.h, z0.h, z1.h \n" |
| "st1h {z1.h}, p0, [%[dst_y]] \n" |
| |
| "99: \n" |
| : [src_y] "+r"(src_y), // %[src_y] |
| [dst_y] "+r"(dst_y), // %[dst_y] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : [scale] "r"(scale) // %[scale] |
| : "memory", "cc", "z0", "z1", "p0"); |
| } |
| |
| __arm_locally_streaming void ARGBMultiplyRow_SME(const uint8_t* src_argb, |
| const uint8_t* src_argb1, |
| uint8_t* dst_argb, |
| int width) { |
| // Streaming-SVE only, no use of ZA tile. |
| width *= 4; |
| int vl; |
| asm volatile( |
| "cntb %x[vl] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p0.b \n" |
| "1: \n" |
| "ld1b {z0.b}, p0/z, [%[src_argb]] \n" |
| "ld1b {z1.b}, p0/z, [%[src_argb1]] \n" |
| "incb %[src_argb] \n" |
| "incb %[src_argb1] \n" |
| "umullb z2.h, z0.b, z1.b \n" |
| "umullt z1.h, z0.b, z1.b \n" |
| "rshrnb z0.b, z2.h, #8 \n" |
| "rshrnt z0.b, z1.h, #8 \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "st1b {z0.b}, p0, [%[dst_argb]] \n" |
| "incb %[dst_argb] \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| "whilelt p0.b, wzr, %w[width] \n" |
| "ld1b {z0.b}, p0/z, [%[src_argb]] \n" |
| "ld1b {z1.b}, p0/z, [%[src_argb1]] \n" |
| "umullb z2.h, z0.b, z1.b \n" |
| "umullt z1.h, z0.b, z1.b \n" |
| "rshrnb z0.b, z2.h, #8 \n" |
| "rshrnt z0.b, z1.h, #8 \n" |
| "st1b {z0.b}, p0, [%[dst_argb]] \n" |
| |
| "99: \n" |
| : [src_argb] "+r"(src_argb), // %[src_argb] |
| [src_argb1] "+r"(src_argb1), // %[src_argb1] |
| [dst_argb] "+r"(dst_argb), // %[dst_argb] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : |
| : "memory", "cc", "z0", "z1", "z2", "p0", "p1"); |
| } |
| |
| __arm_locally_streaming void MergeUVRow_SME(const uint8_t* src_u, |
| const uint8_t* src_v, |
| uint8_t* dst_uv, |
| int width) { |
| // Streaming-SVE only, no use of ZA tile. |
| int vl; |
| asm volatile( |
| "cntb %x[vl] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p0.b \n" |
| "1: \n" |
| "ld1b {z1.b}, p0/z, [%[src_u]] \n" |
| "ld1b {z2.b}, p0/z, [%[src_v]] \n" |
| "incb %[src_u] \n" |
| "incb %[src_v] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" |
| "incb %[dst_uv], all, mul #2 \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| "whilelt p0.b, wzr, %w[width] \n" |
| "ld1b {z1.b}, p0/z, [%[src_u]] \n" |
| "ld1b {z2.b}, p0/z, [%[src_v]] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n" |
| |
| "99: \n" |
| : [src_u] "+r"(src_u), // %[src_u] |
| [src_v] "+r"(src_v), // %[src_v] |
| [dst_uv] "+r"(dst_uv), // %[dst_uv] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : |
| : "memory", "cc", "z0", "z1", "z2", "p0"); |
| } |
| |
| __arm_locally_streaming void MergeUVRow_16_SME(const uint16_t* src_u, |
| const uint16_t* src_v, |
| uint16_t* dst_uv, |
| int depth, |
| int width) { |
| int shift = 16 - depth; |
| // Streaming-SVE only, no use of ZA tile. |
| int vl; |
| asm volatile( |
| "cnth %x[vl] \n" |
| "mov z0.h, %w[shift] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p0.h \n" |
| "1: \n" |
| "ld1h {z1.h}, p0/z, [%[src_u]] \n" |
| "ld1h {z2.h}, p0/z, [%[src_v]] \n" |
| "incb %[src_u] \n" |
| "incb %[src_v] \n" |
| "lsl z1.h, p0/m, z1.h, z0.h \n" |
| "lsl z2.h, p0/m, z2.h, z0.h \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "st2h {z1.h, z2.h}, p0, [%[dst_uv]] \n" |
| "incb %[dst_uv], all, mul #2 \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| "whilelt p0.h, wzr, %w[width] \n" |
| "ld1h {z1.h}, p0/z, [%[src_u]] \n" |
| "ld1h {z2.h}, p0/z, [%[src_v]] \n" |
| "lsl z1.h, p0/m, z1.h, z0.h \n" |
| "lsl z2.h, p0/m, z2.h, z0.h \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "st2h {z1.h, z2.h}, p0, [%[dst_uv]] \n" |
| |
| "99: \n" |
| : [src_u] "+r"(src_u), // %[src_u] |
| [src_v] "+r"(src_v), // %[src_v] |
| [dst_uv] "+r"(dst_uv), // %[dst_uv] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : [shift] "r"(shift) // %[shift] |
| : "memory", "cc", "z0", "z1", "z2", "p0"); |
| } |
| |
| // Use scale to convert lsb formats to msb, depending how many bits there are: |
| // 32768 = 9 bits = shr 1 |
| // 16384 = 10 bits = shr 2 |
| // 4096 = 12 bits = shr 4 |
| // 256 = 16 bits = shr 8 |
| __arm_locally_streaming void Convert16To8Row_SME(const uint16_t* src_y, |
| uint8_t* dst_y, |
| int scale, |
| int width) { |
| // 15 - clz(scale), + 8 to shift result into the high half of the lane to |
| // saturate, then we can just use UZP2 to narrow rather than a pair of |
| // saturating narrow instructions. |
| const int shift = 23 - __builtin_clz((int32_t)scale); |
| int vl; |
| asm volatile( |
| "cntb %x[vl] \n" |
| "dup z0.h, %w[shift] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p0.b \n" |
| "1: \n" |
| "ld1h {z1.h}, p0/z, [%[src_y]] \n" |
| "ld1h {z2.h}, p0/z, [%[src_y], #1, mul vl] \n" |
| "incb %[src_y], all, mul #2 \n" |
| "uqshl z1.h, p0/m, z1.h, z0.h \n" |
| "uqshl z2.h, p0/m, z2.h, z0.h \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "uzp2 z1.b, z1.b, z2.b \n" |
| "st1b {z1.b}, p0, [%[dst_y]] \n" |
| "incb %[dst_y] \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| // We need separate predicates for the load and store instructions since |
| // they are operating on different element sizes (.b vs .h). |
| "cnth %x[vl] \n" |
| "whilelt p0.h, wzr, %w[width] \n" |
| "whilelt p1.h, %w[vl], %w[width] \n" |
| "whilelt p2.b, wzr, %w[width] \n" |
| "ld1h {z1.h}, p0/z, [%[src_y]] \n" |
| "ld1h {z2.h}, p1/z, [%[src_y], #1, mul vl] \n" |
| "uqshl z1.h, p0/m, z1.h, z0.h \n" |
| "uqshl z2.h, p1/m, z2.h, z0.h \n" |
| "uzp2 z1.b, z1.b, z2.b \n" |
| "st1b {z1.b}, p2, [%[dst_y]] \n" |
| |
| "99: \n" |
| : [src_y] "+r"(src_y), // %[src_y] |
| [dst_y] "+r"(dst_y), // %[dst_y] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : [shift] "r"(shift) // %[shift] |
| : "cc", "memory", "z0", "z1", "z2", "p0", "p1", "p2"); |
| } |
| |
| __arm_locally_streaming void CopyRow_SME(const uint8_t* src, |
| uint8_t* dst, |
| int width) { |
| // Streaming-SVE only, no use of ZA tile. |
| int vl; |
| asm volatile( |
| "cntb %x[vl] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p0.b \n" |
| "1: \n" |
| "ld1b {z0.b}, p0/z, [%[src]] \n" |
| "incb %[src] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "st1b {z0.b}, p0, [%[dst]] \n" |
| "incb %[dst] \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| "whilelt p0.b, wzr, %w[width] \n" |
| "ld1b {z0.b}, p0/z, [%[src]] \n" |
| "st1b {z0.b}, p0, [%[dst]] \n" |
| |
| "99: \n" |
| : [src] "+r"(src), // %[src] |
| [dst] "+r"(dst), // %[dst] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : |
| : "memory", "cc", "z0", "p0"); |
| } |
| |
| __arm_locally_streaming static void HalfRow_SME(uint8_t* dst_ptr, |
| const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| int width) { |
| const uint8_t* src_ptr1 = src_ptr + src_stride; |
| |
| int vl; |
| asm volatile( |
| "cntb %x[vl] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p0.b \n" |
| "1: \n" |
| "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" |
| "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" |
| "incb %[src_ptr] \n" |
| "incb %[src_ptr1] \n" |
| "urhadd z2.b, p0/m, z2.b, z3.b \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "st1b {z2.b}, p0, [%[dst_ptr]] \n" |
| "incb %[dst_ptr] \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| "whilelt p0.b, wzr, %w[width] \n" |
| "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" |
| "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" |
| "urhadd z2.b, p0/m, z2.b, z3.b \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "st1b {z2.b}, p0, [%[dst_ptr]] \n" |
| |
| "99: \n" |
| : [src_ptr] "+r"(src_ptr), // %[src_ptr] |
| [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] |
| [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : |
| : "cc", "memory", "z0", "z1", "z2", "z3", "p0"); |
| } |
| |
| __arm_locally_streaming void InterpolateRow_SME(uint8_t* dst_ptr, |
| const uint8_t* src_ptr, |
| ptrdiff_t src_stride, |
| int width, |
| int source_y_fraction) { |
| int y1_fraction = source_y_fraction; |
| int y0_fraction = 256 - y1_fraction; |
| const uint8_t* src_ptr1 = src_ptr + src_stride; |
| |
| if (y0_fraction == 0) { |
| CopyRow_SME(src_ptr1, dst_ptr, width); |
| return; |
| } |
| if (y0_fraction == 128) { |
| HalfRow_SME(dst_ptr, src_ptr, src_stride, width); |
| return; |
| } |
| if (y0_fraction == 256) { |
| CopyRow_SME(src_ptr, dst_ptr, width); |
| return; |
| } |
| |
| int vl; |
| asm volatile( |
| "cntb %x[vl] \n" |
| "dup z0.b, %w[y0_fraction] \n" |
| "dup z1.b, %w[y1_fraction] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p0.b \n" |
| "1: \n" |
| "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" |
| "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" |
| "incb %[src_ptr] \n" |
| "incb %[src_ptr1] \n" |
| "umullb z4.h, z2.b, z0.b \n" |
| "umullt z2.h, z2.b, z0.b \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "umlalb z4.h, z3.b, z1.b \n" |
| "umlalt z2.h, z3.b, z1.b \n" |
| "rshrnb z3.b, z4.h, #8 \n" |
| "rshrnt z3.b, z2.h, #8 \n" |
| "st1b {z3.b}, p0, [%[dst_ptr]] \n" |
| "incb %[dst_ptr] \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| "whilelt p0.b, wzr, %w[width] \n" |
| "ld1b {z2.b}, p0/z, [%[src_ptr]] \n" |
| "ld1b {z3.b}, p0/z, [%[src_ptr1]] \n" |
| "umullb z4.h, z2.b, z0.b \n" |
| "umullt z2.h, z2.b, z0.b \n" |
| "umlalb z4.h, z3.b, z1.b \n" |
| "umlalt z2.h, z3.b, z1.b \n" |
| "rshrnb z3.b, z4.h, #8 \n" |
| "rshrnt z3.b, z2.h, #8 \n" |
| "st1b {z3.b}, p0, [%[dst_ptr]] \n" |
| |
| "99: \n" |
| : [src_ptr] "+r"(src_ptr), // %[src_ptr] |
| [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] |
| [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] |
| [y1_fraction] "r"(y1_fraction) // %[y1_fraction] |
| : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0"); |
| } |
| |
| __arm_locally_streaming static void HalfRow_16_SME(uint16_t* dst_ptr, |
| const uint16_t* src_ptr, |
| ptrdiff_t src_stride, |
| int width) { |
| const uint16_t* src_ptr1 = src_ptr + src_stride; |
| |
| int vl; |
| asm volatile( |
| "cnth %x[vl] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p0.h \n" |
| "1: \n" |
| "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" |
| "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" |
| "incb %[src_ptr] \n" |
| "incb %[src_ptr1] \n" |
| "urhadd z2.h, p0/m, z2.h, z3.h \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "st1h {z2.h}, p0, [%[dst_ptr]] \n" |
| "incb %[dst_ptr] \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| "whilelt p0.h, wzr, %w[width] \n" |
| "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" |
| "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" |
| "urhadd z2.h, p0/m, z2.h, z3.h \n" |
| "st1h {z2.h}, p0, [%[dst_ptr]] \n" |
| |
| "99: \n" |
| : [src_ptr] "+r"(src_ptr), // %[src_ptr] |
| [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] |
| [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : |
| : "cc", "memory", "z0", "z1", "z2", "z3", "p0"); |
| } |
| |
| __arm_locally_streaming void InterpolateRow_16_SME(uint16_t* dst_ptr, |
| const uint16_t* src_ptr, |
| ptrdiff_t src_stride, |
| int width, |
| int source_y_fraction) { |
| int y1_fraction = source_y_fraction; |
| int y0_fraction = 256 - y1_fraction; |
| const uint16_t* src_ptr1 = src_ptr + src_stride; |
| |
| if (y0_fraction == 0) { |
| CopyRow_SME((const uint8_t*)src_ptr1, (uint8_t*)dst_ptr, |
| width * sizeof(uint16_t)); |
| return; |
| } |
| if (y0_fraction == 128) { |
| HalfRow_16_SME(dst_ptr, src_ptr, src_stride, width); |
| return; |
| } |
| if (y0_fraction == 256) { |
| CopyRow_SME((const uint8_t*)src_ptr, (uint8_t*)dst_ptr, |
| width * sizeof(uint16_t)); |
| return; |
| } |
| |
| int vl; |
| asm volatile( |
| "cnth %x[vl] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "dup z0.h, %w[y0_fraction] \n" |
| "dup z1.h, %w[y1_fraction] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p0.h \n" |
| "1: \n" |
| "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" |
| "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" |
| "incb %[src_ptr] \n" |
| "incb %[src_ptr1] \n" |
| "umullb z4.s, z2.h, z0.h \n" |
| "umullt z2.s, z2.h, z0.h \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "umlalb z4.s, z3.h, z1.h \n" |
| "umlalt z2.s, z3.h, z1.h \n" |
| "rshrnb z3.h, z4.s, #8 \n" |
| "rshrnt z3.h, z2.s, #8 \n" |
| "st1h {z3.h}, p0, [%[dst_ptr]] \n" |
| "incb %[dst_ptr] \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| "whilelt p0.h, wzr, %w[width] \n" |
| "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" |
| "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" |
| "umullb z4.s, z2.h, z0.h \n" |
| "umullt z2.s, z2.h, z0.h \n" |
| "umlalb z4.s, z3.h, z1.h \n" |
| "umlalt z2.s, z3.h, z1.h \n" |
| "rshrnb z3.h, z4.s, #8 \n" |
| "rshrnt z3.h, z2.s, #8 \n" |
| "st1h {z3.h}, p0, [%[dst_ptr]] \n" |
| |
| "99: \n" |
| : [src_ptr] "+r"(src_ptr), // %[src_ptr] |
| [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] |
| [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] |
| [y1_fraction] "r"(y1_fraction) // %[y1_fraction] |
| : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "p0"); |
| } |
| |
| __arm_locally_streaming static void HalfRow_16To8_SME(uint8_t* dst_ptr, |
| const uint16_t* src_ptr, |
| ptrdiff_t src_stride, |
| int scale, |
| int width) { |
| const uint16_t* src_ptr1 = src_ptr + src_stride; |
| |
| // 15 - clz(scale), + 8 to shift result into the high half of the lane to |
| // saturate, then we can just use UZP2 to narrow rather than a pair of |
| // saturating narrow instructions. |
| const int shift = 23 - __builtin_clz((int32_t)scale); |
| |
| int vl; |
| asm volatile( |
| "cnth %x[vl] \n" |
| "dup z31.h, %w[shift] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p0.h \n" |
| "1: \n" |
| "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" |
| "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" |
| "incb %[src_ptr] \n" |
| "incb %[src_ptr1] \n" |
| "urhadd z2.h, p0/m, z2.h, z3.h \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "uqshl z2.h, p0/m, z2.h, z31.h \n" |
| "shrnb z2.b, z2.h, #8 \n" |
| "st1b {z2.h}, p0, [%[dst_ptr]] \n" |
| "inch %[dst_ptr] \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| "whilelt p0.h, wzr, %w[width] \n" |
| "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" |
| "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" |
| "urhadd z2.h, p0/m, z2.h, z3.h \n" |
| "uqshl z2.h, p0/m, z2.h, z31.h \n" |
| "shrnb z2.b, z2.h, #8 \n" |
| "st1b {z2.h}, p0, [%[dst_ptr]] \n" |
| |
| "99: \n" |
| : [src_ptr] "+r"(src_ptr), // %[src_ptr] |
| [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] |
| [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : [shift] "r"(shift) // %[shift] |
| : "cc", "memory", "z0", "z1", "z2", "z3", "z31", "p0"); |
| } |
| |
| // Use scale to convert lsb formats to msb, depending how many bits there are: |
| // 32768 = 9 bits |
| // 16384 = 10 bits |
| // 4096 = 12 bits |
| // 256 = 16 bits |
| // TODO(fbarchard): change scale to bits |
| __arm_locally_streaming void InterpolateRow_16To8_SME(uint8_t* dst_ptr, |
| const uint16_t* src_ptr, |
| ptrdiff_t src_stride, |
| int scale, |
| int width, |
| int source_y_fraction) { |
| const int y1_fraction = source_y_fraction; |
| const int y0_fraction = 256 - y1_fraction; |
| const uint16_t* src_ptr1 = src_ptr + src_stride; |
| |
| // y0_fraction == 0 is never called here. |
| if (y0_fraction == 128) { |
| HalfRow_16To8_SME(dst_ptr, src_ptr, src_stride, scale, width); |
| return; |
| } |
| if (y0_fraction == 256) { |
| Convert16To8Row_SME(src_ptr, dst_ptr, scale, width); |
| return; |
| } |
| |
| // 15 - clz(scale), + 8 to shift result into the high half of the lane to |
| // saturate, then we can just use UZP2 to narrow rather than a pair of |
| // saturating narrow instructions. |
| const int shift = 23 - __builtin_clz((int32_t)scale); |
| |
| int vl; |
| asm volatile( |
| "cnth %x[vl] \n" |
| "dup z31.h, %w[shift] \n" |
| "dup z0.h, %w[y0_fraction] \n" |
| "dup z1.h, %w[y1_fraction] \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with an all-true predicate to avoid predicate |
| // generation overhead. |
| "ptrue p0.h \n" |
| "1: \n" |
| "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" |
| "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" |
| "incb %[src_ptr] \n" |
| "incb %[src_ptr1] \n" |
| "umullb z4.s, z2.h, z0.h \n" |
| "umullt z2.s, z2.h, z0.h \n" |
| "subs %w[width], %w[width], %w[vl] \n" |
| "umlalb z4.s, z3.h, z1.h \n" |
| "umlalt z2.s, z3.h, z1.h \n" |
| "rshrnb z3.h, z4.s, #8 \n" |
| "rshrnt z3.h, z2.s, #8 \n" |
| "uqshl z3.h, p0/m, z3.h, z31.h \n" |
| "shrnb z3.b, z3.h, #8 \n" |
| "st1b {z3.h}, p0, [%[dst_ptr]] \n" |
| "inch %[dst_ptr] \n" |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl] \n" |
| "b.eq 99f \n" |
| |
| // Calculate a predicate for the final iteration to deal with the tail. |
| "whilelt p0.h, wzr, %w[width] \n" |
| "ld1h {z2.h}, p0/z, [%[src_ptr]] \n" |
| "ld1h {z3.h}, p0/z, [%[src_ptr1]] \n" |
| "umullb z4.s, z2.h, z0.h \n" |
| "umullt z2.s, z2.h, z0.h \n" |
| "umlalb z4.s, z3.h, z1.h \n" |
| "umlalt z2.s, z3.h, z1.h \n" |
| "rshrnb z3.h, z4.s, #8 \n" |
| "rshrnt z3.h, z2.s, #8 \n" |
| "uqshl z3.h, p0/m, z3.h, z31.h \n" |
| "shrnb z3.b, z3.h, #8 \n" |
| "st1b {z3.h}, p0, [%[dst_ptr]] \n" |
| |
| "99: \n" |
| : [src_ptr] "+r"(src_ptr), // %[src_ptr] |
| [src_ptr1] "+r"(src_ptr1), // %[src_ptr1] |
| [dst_ptr] "+r"(dst_ptr), // %[dst_ptr] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : [y0_fraction] "r"(y0_fraction), // %[y0_fraction] |
| [y1_fraction] "r"(y1_fraction), // %[y1_fraction] |
| [shift] "r"(shift) // %[shift] |
| : "cc", "memory", "z0", "z1", "z2", "z3", "z4", "z31", "p0"); |
| } |
| |
| __arm_locally_streaming void Convert8To8Row_SME(const uint8_t* src_y, |
| uint8_t* dst_y, |
| int scale, |
| int bias, |
| int width) { |
| Convert8To8Row_SVE_SC(src_y, dst_y, scale, bias, width); |
| } |
| |
| #define CONVERT8TO16_SVE \ |
| "ld1b {z0.h}, p0/z, [%[src]] \n" \ |
| "ld1b {z1.h}, p1/z, [%[src], #1, mul vl] \n" \ |
| "incb %[src] \n" \ |
| "subs %w[width], %w[width], %w[vl], lsl #1 \n" \ |
| "trn1 z0.b, z0.b, z0.b \n" \ |
| "trn1 z1.b, z1.b, z1.b \n" \ |
| "lsr z0.h, p0/m, z0.h, z2.h \n" \ |
| "lsr z1.h, p1/m, z1.h, z2.h \n" \ |
| "prfm pldl1keep, [%[src], 448] \n" \ |
| "st1h {z0.h}, p0, [%[dst]] \n" \ |
| "st1h {z1.h}, p1, [%[dst], #1, mul vl] \n" \ |
| "incb %[dst], all, mul #2 \n" |
| |
| __arm_locally_streaming void Convert8To16Row_SME(const uint8_t* src_y, |
| uint16_t* dst_y, |
| int scale, |
| int width) { |
| // (src * 0x0101 * scale) >> 16. |
| // Since scale is a power of two, compute the shift to use to avoid needing |
| // to widen to int32. |
| const int shift = __builtin_clz(scale) - 15; |
| |
| uint64_t vl; |
| asm volatile( |
| "dup z2.h, %w[shift] \n" |
| "cnth %[vl] \n" |
| "subs %w[width], %w[width], %w[vl], lsl #1 \n" |
| "b.lt 2f \n" |
| |
| // Run bulk of computation with all-true predicates to avoid predicate |
| // generation overhead. |
| "ptrue p0.h \n" |
| "ptrue p1.h \n" |
| "1: \n" // |
| CONVERT8TO16_SVE |
| "b.ge 1b \n" |
| |
| "2: \n" |
| "adds %w[width], %w[width], %w[vl], lsl #1 \n" |
| "b.eq 99f \n" |
| |
| // Calculate predicates for the final iteration to deal with the tail. |
| "whilelt p0.h, wzr, %w[width] \n" |
| "whilelt p1.h, %w[vl], %w[width] \n" // |
| CONVERT8TO16_SVE |
| |
| "99: \n" |
| : [src] "+r"(src_y), // %[src] |
| [dst] "+r"(dst_y), // %[dst] |
| [width] "+r"(width), // %[width] |
| [vl] "=&r"(vl) // %[vl] |
| : [shift] "r"(shift) // %[shift] |
| : "cc", "memory", "z0", "z1", "z2", "p0", "p1"); |
| } |
| |
| __arm_locally_streaming void ARGBToUVRow_SME(const uint8_t* src_argb, |
| int src_stride_argb, |
| uint8_t* dst_u, |
| uint8_t* dst_v, |
| int width) { |
| ARGBToUVMatrixRow_SVE_SC(src_argb, src_stride_argb, dst_u, dst_v, width, |
| kARGBToUVCoefficients); |
| } |
| |
| __arm_locally_streaming void ARGBToUVJRow_SME(const uint8_t* src_argb, |
| int src_stride_argb, |
| uint8_t* dst_u, |
| uint8_t* dst_v, |
| int width) { |
| ARGBToUVMatrixRow_SVE_SC(src_argb, src_stride_argb, dst_u, dst_v, width, |
| kARGBToUVJCoefficients); |
| } |
| |
| __arm_locally_streaming void ABGRToUVJRow_SME(const uint8_t* src_abgr, |
| int src_stride_abgr, |
| uint8_t* dst_uj, |
| uint8_t* dst_vj, |
| int width) { |
| ARGBToUVMatrixRow_SVE_SC(src_abgr, src_stride_abgr, dst_uj, dst_vj, width, |
| kABGRToUVJCoefficients); |
| } |
| |
| __arm_locally_streaming void BGRAToUVRow_SME(const uint8_t* src_bgra, |
| int src_stride_bgra, |
| uint8_t* dst_u, |
| uint8_t* dst_v, |
| int width) { |
| ARGBToUVMatrixRow_SVE_SC(src_bgra, src_stride_bgra, dst_u, dst_v, width, |
| kBGRAToUVCoefficients); |
| } |
| |
| __arm_locally_streaming void ABGRToUVRow_SME(const uint8_t* src_abgr, |
| int src_stride_abgr, |
| uint8_t* dst_u, |
| uint8_t* dst_v, |
| int width) { |
| ARGBToUVMatrixRow_SVE_SC(src_abgr, src_stride_abgr, dst_u, dst_v, width, |
| kABGRToUVCoefficients); |
| } |
| |
| __arm_locally_streaming void RGBAToUVRow_SME(const uint8_t* src_rgba, |
| int src_stride_rgba, |
| uint8_t* dst_u, |
| uint8_t* dst_v, |
| int width) { |
| ARGBToUVMatrixRow_SVE_SC(src_rgba, src_stride_rgba, dst_u, dst_v, width, |
| kRGBAToUVCoefficients); |
| } |
| |
| #endif // !defined(LIBYUV_DISABLE_SME) && defined(CLANG_HAS_SME) && |
| // defined(__aarch64__) |
| |
| #ifdef __cplusplus |
| } // extern "C" |
| } // namespace libyuv |
| #endif |
