[newlib-cygwin/main] RISC-V: memmove() speed optimized: Align source address
Kito Cheng
kito@sourceware.org
Thu Jul 10 01:31:43 GMT 2025
https://sourceware.org/git/gitweb.cgi?p=newlib-cygwin.git;h=a17e73675fd5b161b754b56e9d341705ecd19265
commit a17e73675fd5b161b754b56e9d341705ecd19265
Author: m fally <marlene.fally@gmail.com>
Date: Tue Jun 17 17:07:33 2025 +0200
RISC-V: memmove() speed optimized: Align source address
If misaligned accesses are slow or prohibited, either
source or destination address are unaligned and the number
of bytes to be copied is > SZREG*2, align the source address
to xlen. This speeds up the function in the case where at
least one address is unaligned, since now one word (or doubleword
for rv64) is loaded at a time, therefore reducing the amount of
memory accesses necessary.
We still need to store back individual bytes since the
destination address might (still) be unaligned after aligning
the source.
The threshold of SZREG*2 was chosen to keep the negative effect
on shorter copies caused by the additional overhead from aligning
the source low.
This change also affects the case where both adresses are xlen-
aligned, the memory areas overlap destructively, and length is not
a multiple of SZREG. In the destructive-overlap case, the copying
needs to be done in reversed order. Therefore the length is added
to the addresses first, which causes them to become unaligned.
Reviewed-by: Christian Herber <christian.herber@oss.nxp.com>
Signed-off-by: m fally <marlene.fally@gmail.com>
Diff:
---
newlib/libc/machine/riscv/memmove.c | 185 ++++++++++++++++++++++++++----------
1 file changed, 135 insertions(+), 50 deletions(-)
diff --git a/newlib/libc/machine/riscv/memmove.c b/newlib/libc/machine/riscv/memmove.c
index f8937c2a2..12010f20f 100644
--- a/newlib/libc/machine/riscv/memmove.c
+++ b/newlib/libc/machine/riscv/memmove.c
@@ -1,4 +1,5 @@
/* Copyright (c) 2019 SiFive Inc. All rights reserved.
+ Copyright (c) 2025 Marlene Fally <marlene.fally@gmail.com>
This copyrighted material is made available to anyone wishing to use,
modify, copy, or redistribute it subject to the terms and conditions
@@ -30,6 +31,26 @@ __libc_fast_xlen_aligned (void *dst, const void *src)
#endif
}
+#if !defined(__riscv_misaligned_fast)
+static inline void
+__libc_memmove_misaligned_copy (unsigned char *dst,
+ const uintxlen_t *aligned_src)
+{
+ uintxlen_t src_xlen = *aligned_src;
+
+ *dst++ = (unsigned char)(src_xlen);
+ *dst++ = (unsigned char)(src_xlen >> 8);
+ *dst++ = (unsigned char)(src_xlen >> 16);
+ *dst++ = (unsigned char)(src_xlen >> 24);
+#if __riscv_xlen == 64
+ *dst++ = (unsigned char)(src_xlen >> 32);
+ *dst++ = (unsigned char)(src_xlen >> 40);
+ *dst++ = (unsigned char)(src_xlen >> 48);
+ *dst++ = (unsigned char)(src_xlen >> 56);
+#endif
+}
+#endif
+
static inline void
__libc_aligned_copy_unrolled (uintxlen_t *aligned_dst,
const uintxlen_t *aligned_src)
@@ -69,74 +90,138 @@ memmove (void *dst_void, const void *src_void, size_t length)
src += length;
dst += length;
- if (length >= SZREG && __libc_fast_xlen_aligned (dst, src))
+ if (length >= SZREG)
{
- aligned_dst = (uintxlen_t *)dst;
- aligned_src = (uintxlen_t *)src;
-
- /* If possible, unroll the word-copy loop by a factor 9 to
- match memcpy. This speeds up the copying process for longer
- lengths while barely degrading performance for lengths < SZREG*9.
- Since we are copying backwards, decrement the addresses
- before copying.
- */
- while (length >= SZREG * 9)
+ if (__libc_fast_xlen_aligned (dst, src))
{
- aligned_dst -= 9;
- aligned_src -= 9;
- __libc_aligned_copy_unrolled (aligned_dst, aligned_src);
- length -= (SZREG * 9);
+ aligned_dst = (uintxlen_t *)dst;
+ aligned_src = (uintxlen_t *)src;
+
+ /* If possible, unroll the word-copy loop by a factor 9 to
+ match memcpy. This speeds up the copying process for longer
+ lengths while barely degrading performance for lengths <
+ SZREG*9. Since we are copying backwards, decrement the
+ addresses before copying.
+ */
+ while (length >= SZREG * 9)
+ {
+ aligned_dst -= 9;
+ aligned_src -= 9;
+ __libc_aligned_copy_unrolled (aligned_dst, aligned_src);
+ length -= (SZREG * 9);
+ }
+
+ while (length >= SZREG)
+ {
+ *--aligned_dst = *--aligned_src;
+ length -= SZREG;
+ }
+
+ /* Pick up any residual with a byte copier. */
+ dst = (unsigned char *)aligned_dst;
+ src = (unsigned char *)aligned_src;
}
-
- while (length >= SZREG)
+#if !defined(__riscv_misaligned_fast)
+ else if (length > (SZREG * 2))
{
- *--aligned_dst = *--aligned_src;
- length -= SZREG;
+ /* At least one address is not xlen-aligned. If
+ misaligned accesses are slow or prohibited,
+ align the src so we can load SZREG bytes at a time.
+ This reduces the amount of memory accesses made
+ and therefore improves performance.
+ */
+ while ((uintxlen_t)src & (SZREG - 1))
+ {
+ *--dst = *--src;
+ length--;
+ }
+
+ aligned_src = (uintxlen_t *)src;
+
+ /* Decrement the addresses before copying since
+ we are copying backwards. */
+ do
+ {
+ aligned_src--;
+ dst -= SZREG;
+ __libc_memmove_misaligned_copy (dst, aligned_src);
+ length -= SZREG;
+ }
+ while (length >= SZREG);
+
+ /* Pick up any residual with a byte copier. */
+ src = (unsigned char *)aligned_src;
}
-
- /* Pick up any residual with a byte copier. */
- dst = (unsigned char *)aligned_dst;
- src = (unsigned char *)aligned_src;
+#endif
}
-
while (length--)
{
*--dst = *--src;
}
}
- else
+ else /* Memory areas overlap non-destructively or not at all. */
{
- /* Use optimizing algorithm for a non-destructive copy to closely
- match memcpy. If the size is small or either SRC or DST is unaligned,
- then punt into the byte copy loop. This should be rare. */
- if (length >= SZREG && __libc_fast_xlen_aligned (dst, src))
+ if (length >= SZREG)
{
- aligned_dst = (uintxlen_t *)dst;
- aligned_src = (uintxlen_t *)src;
-
- /* If possible, unroll the word-copy loop by a factor 9 to
- match memcpy. This speeds up the copying process for longer
- lengths while barely degrading performance for lengths < SZREG*9.
- */
- while (length >= SZREG * 9)
+ if (__libc_fast_xlen_aligned (dst, src))
{
- __libc_aligned_copy_unrolled (aligned_dst, aligned_src);
- aligned_dst += 9;
- aligned_src += 9;
- length -= (SZREG * 9);
+ aligned_dst = (uintxlen_t *)dst;
+ aligned_src = (uintxlen_t *)src;
+
+ /* If possible, unroll the word-copy loop by a factor 9 to
+ match memcpy. This speeds up the copying process for longer
+ lengths while barely degrading performance for lengths <
+ SZREG*9.
+ */
+ while (length >= SZREG * 9)
+ {
+ __libc_aligned_copy_unrolled (aligned_dst, aligned_src);
+ aligned_dst += 9;
+ aligned_src += 9;
+ length -= (SZREG * 9);
+ }
+
+ while (length >= SZREG)
+ {
+ *aligned_dst++ = *aligned_src++;
+ length -= SZREG;
+ }
+
+ /* Pick up any residual with a byte copier. */
+ dst = (unsigned char *)aligned_dst;
+ src = (unsigned char *)aligned_src;
}
-
- while (length >= SZREG)
+#if !defined(__riscv_misaligned_fast)
+ else if (length > (SZREG * 2))
{
- *aligned_dst++ = *aligned_src++;
- length -= SZREG;
+ /* At least one address is not xlen-aligned. If
+ misaligned accesses are slow or prohibited,
+ align the src so we can load SZREG bytes at a time.
+ This reduces the amount of memory accesses made
+ and therefore improves performance.
+ */
+ while ((uintxlen_t)src & (SZREG - 1))
+ {
+ *dst++ = *src++;
+ length--;
+ }
+
+ aligned_src = (uintxlen_t *)src;
+
+ do
+ {
+ __libc_memmove_misaligned_copy (dst, aligned_src);
+ aligned_src++;
+ dst += SZREG;
+ length -= SZREG;
+ }
+ while (length >= SZREG);
+
+ /* Pick up any residual with a byte copier. */
+ src = (unsigned char *)aligned_src;
}
-
- /* Pick up any residual with a byte copier. */
- dst = (unsigned char *)aligned_dst;
- src = (unsigned char *)aligned_src;
+#endif
}
-
while (length--)
{
*dst++ = *src++;
More information about the Newlib-cvs
mailing list