AArch64: Fix HalfMergeUVRow_SVE2 average calculation
The existing SVE2 path only computes the average for a pair of input
values instead of groups of four as done in the Neon implementation.
The average of two values must be retained for the final odd element if
present. Existing code just handles this by falling back to the C
implementation, so mirror that for SVE2 as well.
Reduction in time taken relative to the existing Neon code, the SVE2
code remains faster even with the correct averaging:
Cortex-A510: -12.66%
Lumex C1-Nano: -6.38%
Cortex-A720: -9.73%
Lumex C1-Pro: +0.99%
Cortex-X4: -3.05%
Lumex C1-Ultra: -18.98%
Bug: 546962730
Change-Id: I7ab348e64a9d079b81d070cea5fd575243b037d2
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8252281
Reviewed-by: Wan-Teh Chang <wtc@google.com>
Reviewed-by: Justin Green <greenjustin@google.com>
Commit-Queue: Wan-Teh Chang <wtc@google.com>
diff --git a/include/libyuv/row.h b/include/libyuv/row.h
index 736d2f9..98e3323 100644
--- a/include/libyuv/row.h
+++ b/include/libyuv/row.h
@@ -634,7 +634,7 @@
#define HAS_COPYROW_SVE2
#define HAS_DIVIDEROW_16_SVE2
#define HAS_HALFFLOATROW_SVE2
-// #define HAS_HALFMERGEUVROW_SVE2 // Disabled: Bug 546962730
+#define HAS_HALFMERGEUVROW_SVE2
// #define HAS_HALFROW_16TO8_SVE2 // Disabled: NEON version is faster for 128 bit vectors.
// #define HAS_HALFWIDTHROW_16TO8_SVE2 // Disabled: NEON version is faster for 128 bit vectors.
#define HAS_I210ALPHATOARGBROW_SVE2
diff --git a/source/planar_functions.cc b/source/planar_functions.cc
index b7ae2a3..f03c8fa 100644
--- a/source/planar_functions.cc
+++ b/source/planar_functions.cc
@@ -5921,7 +5921,10 @@
}
#endif
#if defined(HAS_HALFMERGEUVROW_SVE2)
- if (TestCpuFlag(kCpuHasSVE2)) {
+ // The average of two values must be retained for the final odd element if
+ // present. Existing code just handles this by falling back to the C
+ // implementation, so mirror that for SVE2 as well.
+ if (TestCpuFlag(kCpuHasSVE2) && IS_ALIGNED(width, 2)) {
HalfMergeUVRow = HalfMergeUVRow_SVE2;
}
#endif
diff --git a/source/row_sve.cc b/source/row_sve.cc
index 4aa66b1..f8fa227 100644
--- a/source/row_sve.cc
+++ b/source/row_sve.cc
@@ -1151,12 +1151,17 @@
"incb %[src_u1] \n"
"incb %[src_v] \n"
"incb %[src_v1] \n"
- "urhadd z1.b, p0/m, z1.b, z2.b \n"
- "urhadd z3.b, p0/m, z3.b, z4.b \n"
- "mov z2.b, p0/m, z3.b \n"
+ "uaddlb z5.h, z1.b, z2.b \n"
+ "uaddlb z6.h, z3.b, z4.b \n"
+ "uaddlt z1.h, z1.b, z2.b \n"
+ "uaddlt z3.h, z3.b, z4.b \n"
+ "add z1.h, z1.h, z5.h \n"
+ "add z3.h, z3.h, z6.h \n"
+ "rshrnb z1.b, z1.h, #2 \n"
+ "rshrnt z1.b, z3.h, #2 \n"
"subs %w[width], %w[width], %w[vl] \n"
- "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
- "incb %[dst_uv], all, mul #2 \n"
+ "st1b {z1.b}, p0, [%[dst_uv]] \n"
+ "incb %[dst_uv] \n"
"b.ge 1b \n"
"2: \n"
@@ -1168,10 +1173,15 @@
"ld1b {z2.b}, p0/z, [%[src_u1]] \n"
"ld1b {z3.b}, p0/z, [%[src_v]] \n"
"ld1b {z4.b}, p0/z, [%[src_v1]] \n"
- "urhadd z1.b, p0/m, z1.b, z2.b \n"
- "urhadd z3.b, p0/m, z3.b, z4.b \n"
- "mov z2.b, p0/m, z3.b \n"
- "st2b {z1.b, z2.b}, p0, [%[dst_uv]] \n"
+ "uaddlb z5.h, z1.b, z2.b \n"
+ "uaddlb z6.h, z3.b, z4.b \n"
+ "uaddlt z1.h, z1.b, z2.b \n"
+ "uaddlt z3.h, z3.b, z4.b \n"
+ "add z1.h, z1.h, z5.h \n"
+ "add z3.h, z3.h, z6.h \n"
+ "rshrnb z1.b, z1.h, #2 \n"
+ "rshrnt z1.b, z3.h, #2 \n"
+ "st1b {z1.b}, p0, [%[dst_uv]] \n"
"99: \n"
: [src_u] "+r"(src_u), // %[src_u]
@@ -1182,7 +1192,7 @@
[width] "+r"(width), // %[width]
[vl] "=&r"(vl) // %[vl]
:
- : "cc", "memory", "z1", "z2", "z3", "z4", "p0");
+ : "cc", "memory", "z1", "z2", "z3", "z4", "z5", "z6", "p0");
}
void CopyRow_SVE2(const uint8_t* src, uint8_t* dst, int width) {
