[PATCH] aarch64: Optimize memcpy for Kunpeng 950 using SVE and loop unrolling
Wilco Dijkstra
Wilco.Dijkstra@arm.com
Mon Dec 29 15:00:19 GMT 2025
Hi Weihong,
So overall the results look good. I've pointed out a few minor issues below. However
the main issue is that it won't work with different vector lengths and thus fails the
benchmarks and testsuite on other machines, so that will need to be fixed.
It's clearly better in the random and large benchmarks - the main loop is fastest
from 65536 bytes. However memcpy_sve and memcpy_a64fx are typically faster
from 64 bytes. Since the sizes 64-1024 are important, it is worth checking whether
you can further improve that range. Perhaps a lower alignment, less unrolling or
aligning the destination (which is what memcpy_a64fx does) works out better for
medium sizes. Note this could be done in later patches if needed.
diff --git a/sysdeps/aarch64/cpu-features.h b/sysdeps/aarch64/cpu-features.h
index 855990b575..8330c13884 100644
--- a/sysdeps/aarch64/cpu-features.h
+++ b/sysdeps/aarch64/cpu-features.h
@@ -45,6 +45,9 @@
#define IS_KUNPENG920(midr) (MIDR_IMPLEMENTOR(midr) == 'H' \
&& MIDR_PARTNUM(midr) == 0xd01)
+
+#define IS_KUNPENG950(midr) (MIDR_IMPLEMENTOR(midr) == 'H' \
+ && MIDR_PARTNUM(midr) == 0xd06)
#define IS_A64FX(midr) (MIDR_IMPLEMENTOR(midr) == 'F' \
&& MIDR_PARTNUM(midr) == 0x001)
diff --git a/sysdeps/aarch64/multiarch/Makefile b/sysdeps/aarch64/multiarch/Makefile
index 1c3c392513..96ad9828d2 100644
--- a/sysdeps/aarch64/multiarch/Makefile
+++ b/sysdeps/aarch64/multiarch/Makefile
@@ -3,6 +3,7 @@ sysdep_routines += \
memchr_generic \
memchr_nosimd \
memcpy_a64fx \
+ memcpy_kunpeng950 \
As you already noted, the entries need to be alphabetically sorted.
memcpy_generic \
memcpy_mops \
memcpy_oryon1 \
diff --git a/sysdeps/aarch64/multiarch/ifunc-impl-list.c b/sysdeps/aarch64/multiarch/ifunc-impl-list.c
index 0e26171929..a06c8882e4 100644
--- a/sysdeps/aarch64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/aarch64/multiarch/ifunc-impl-list.c
@@ -38,6 +38,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_oryon1)
IFUNC_IMPL_ADD (array, i, memcpy, sve, __memcpy_a64fx)
IFUNC_IMPL_ADD (array, i, memcpy, sve, __memcpy_sve)
+ IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_kunpeng950)
This needs to use "sve". Currently it will fail benchmarks and the testsuite on
any machine that doesn't have SVE or the same SVE vector length...
Also note indentation.
IFUNC_IMPL_ADD (array, i, memcpy, mops, __memcpy_mops)
IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_generic))
IFUNC_IMPL (i, name, memmove,
diff --git a/sysdeps/aarch64/multiarch/memcpy.c b/sysdeps/aarch64/multiarch/memcpy.c
index 894dabe2ef..d649f22c1c 100644
--- a/sysdeps/aarch64/multiarch/memcpy.c
+++ b/sysdeps/aarch64/multiarch/memcpy.c
@@ -34,12 +34,16 @@ extern __typeof (__redirect_memcpy) __memcpy_a64fx attribute_hidden;
extern __typeof (__redirect_memcpy) __memcpy_sve attribute_hidden;
extern __typeof (__redirect_memcpy) __memcpy_mops attribute_hidden;
extern __typeof (__redirect_memcpy) __memcpy_oryon1 attribute_hidden;
+extern __typeof (__redirect_memcpy) __memcpy_kunpeng950 attribute_hidden;
static inline __typeof (__redirect_memcpy) *
select_memcpy_ifunc (void)
{
INIT_ARCH ();
+ if (IS_KUNPENG950 (midr))
+ return __memcpy_kunpeng950;
+
This should be inside the if (sve) block. You cannot assume the OS always
enables SVE.
diff --git a/sysdeps/aarch64/multiarch/memcpy_kunpeng950.S b/sysdeps/aarch64/multiarch/memcpy_kunpeng950.S
new file mode 100644
index 0000000000..cb03a1f762
--- /dev/null
+++ b/sysdeps/aarch64/multiarch/memcpy_kunpeng950.S
@@ -0,0 +1,137 @@
+/* Optimized glibc function for Huawei Kupeng 950 processor.
+ Copyright (C) 2012-2022 Free Software Foundation, Inc.
Surely 2025?
+ Copyright (c) 2025 Huawei Technologies Co., Ltd.
Does Huawei have a copyright assignment with FSF? If so, you only need the
FSF Copyright here. If not, and it is as an individual Signed-off-by, then I think
you just mention "Copyright The GNU Toolchain Authors." here.
+ENTRY (__memcpy_kunpeng950)
+ cmp cnt, 192
+ b.hi L(192_more)
+ cntb vlen
+
+L(less_64):
+ whilelo p0.b, xzr, cnt
+ whilelo p1.b, vlen, cnt
+ ld1b z0.b, p0/z, [src, 0, mul vl]
+ ld1b z1.b, p1/z, [src, 1, mul vl]
+ st1b z0.b, p0, [dst_in, 0, mul vl]
+ st1b z1.b, p1, [dst_in, 1, mul vl]
+ subs cnt, cnt, 64
This makes the code dependent on vector length...
You can do "subs cnt, cnt, vlen, lsl 1" here (which should not be slower),
but it will need more changes to become vector length agnostic.
+ b.hi L(64_more)
+ ret
+
+L(64_more):
+ whilelo p2.b, xzr, cnt
+ whilelo p3.b, vlen, cnt
+ ld1b z2.b, p2/z, [src, 2, mul vl]
+ ld1b z3.b, p3/z, [src, 3, mul vl]
+ st1b z2.b, p2, [dst_in, 2, mul vl]
+ st1b z3.b, p3, [dst_in, 3, mul vl]
+ subs cnt, cnt, 64
+ b.hi L(128_more)
+ ret
+
+L(128_more):
+ whilelo p4.b, xzr, cnt
+ whilelo p5.b, vlen, cnt
+ ld1b z4.b, p4/z, [src, 4, mul vl]
+ ld1b z5.b, p5/z, [src, 5, mul vl]
+ st1b z4.b, p4, [dst_in, 4, mul vl]
+ st1b z5.b, p5, [dst_in, 5, mul vl]
+ ret
Is it worth aligning so the main loop ends up 16-byte aligned without nops?
+L(192_more):
+ ldp E_q, F_q, [src]
+ ldp G_q, H_q, [src, 32]
+ add src_end, src, cnt
+ add dst_end, dst_in, cnt
+ mov dst, dst_in
+ and tmp1, src, 63
+ cbz tmp1, L(already_align_64)
This adds 2 instructions to skip 3 - and since it's more likely to not be 64-byte
aligned, you usually execute 5 instructions rather than 3...
Also is there a real benefit to 64-byte alignment? LDP might prefer 32-byte
alignment rather than 16, but 64? Also it is worth checking whether aligning
STP instead works out better based on the benchmark results.
+ bic src, src, 63
+ sub dst, dst_in, tmp1
+ add cnt, cnt, tmp1
+L(already_align_64):
+ ldp A_q, B_q, [src, 64]
+ ldp C_q, D_q, [src, 96]
+
+ ldp I_q, J_q, [src_end, -128]
+ ldp K_q, L_q, [src_end, -96]
+ ldp M_q, N_q, [src_end, -64]
+ ldp O_q, P_q, [src_end, -32]
+
+ stp E_q, F_q, [dst_in]
+ stp G_q, H_q, [dst_in, 32]
+ subs cnt, cnt, 128+64+64
+ b.ls L(tail128_align_64)
+
+ .p2align 4
+L(loop128_align_64):
+ ldp E_q, F_q, [src, 128]
+ stp A_q, B_q, [dst, 64]
+ ldp G_q, H_q, [src, 160]
+ stp C_q, D_q, [dst, 96]
+ ldp A_q, B_q, [src, 192]
+ stp E_q, F_q, [dst, 128]
+ ldp C_q, D_q, [src, 224]
+ stp G_q, H_q, [dst, 160]
+
+ add src, src, 128
+ add dst, dst, 128
+ subs cnt, cnt, 128
+ b.hi L(loop128_align_64)
+
+L(tail128_align_64):
+ stp A_q, B_q, [dst, 64]
+ stp C_q, D_q, [dst, 96]
+
+ stp I_q, J_q, [dst_end, -128]
+ stp K_q, L_q, [dst_end, -96]
+ stp M_q, N_q, [dst_end, -64]
+ stp O_q, P_q, [dst_end, -32]
+ ret
+END (__memcpy_kunpeng950)
\ No newline at end of file
Please fix.
Cheers,
Wilco
More information about the Libc-alpha
mailing list