[PATCH, AArch64]: Further strcpy improvements. Add stpcpy.
Richard Earnshaw
rearnsha@arm.com
Tue Jan 6 10:07:00 GMT 2015
This patch is a further performance improvement for short strings using
strcpy. The main changes are to eliminate further conditional branches
that are hard to predict and to apply some simplifications that fall out
of those changes. Furthermore, we are now more likely to fall-through
conditional branches on the likely path, which helps branch prediction
further.
The patch also adds the ability to use the same code to build stpcpy.
Committed.
-------------- next part --------------
? libc/machine/aarch64/autom4te.cache
Index: libc/machine/aarch64/Makefile.am
===================================================================
RCS file: /cvs/src/src/newlib/libc/machine/aarch64/Makefile.am,v
retrieving revision 1.12
retrieving revision 1.13
diff -u -p -r1.12 -r1.13
--- libc/machine/aarch64/Makefile.am 8 Dec 2014 15:21:42 -0000 1.12
+++ libc/machine/aarch64/Makefile.am 6 Jan 2015 09:57:54 -0000 1.13
@@ -20,6 +20,8 @@ lib_a_SOURCES += memmove.S
lib_a_SOURCES += memset-stub.c
lib_a_SOURCES += memset.S
lib_a_SOURCES += setjmp.S
+lib_a_SOURCES += stpcpy-stub.c
+lib_a_SOURCES += stpcpy.S
lib_a_SOURCES += strchr-stub.c
lib_a_SOURCES += strchr.S
lib_a_SOURCES += strchrnul-stub.c
Index: libc/machine/aarch64/Makefile.in
===================================================================
RCS file: /cvs/src/src/newlib/libc/machine/aarch64/Makefile.in,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -p -r1.13 -r1.14
--- libc/machine/aarch64/Makefile.in 8 Dec 2014 15:21:42 -0000 1.13
+++ libc/machine/aarch64/Makefile.in 6 Jan 2015 09:57:55 -0000 1.14
@@ -74,7 +74,8 @@ am_lib_a_OBJECTS = lib_a-memchr-stub.$(O
lib_a-memcpy-stub.$(OBJEXT) lib_a-memcpy.$(OBJEXT) \
lib_a-memmove-stub.$(OBJEXT) lib_a-memmove.$(OBJEXT) \
lib_a-memset-stub.$(OBJEXT) lib_a-memset.$(OBJEXT) \
- lib_a-setjmp.$(OBJEXT) lib_a-strchr-stub.$(OBJEXT) \
+ lib_a-setjmp.$(OBJEXT) lib_a-stpcpy-stub.$(OBJEXT) \
+ lib_a-stpcpy.$(OBJEXT) lib_a-strchr-stub.$(OBJEXT) \
lib_a-strchr.$(OBJEXT) lib_a-strchrnul-stub.$(OBJEXT) \
lib_a-strchrnul.$(OBJEXT) lib_a-strcmp-stub.$(OBJEXT) \
lib_a-strcmp.$(OBJEXT) lib_a-strcpy-stub.$(OBJEXT) \
@@ -210,10 +211,10 @@ AM_CCASFLAGS = $(INCLUDES)
noinst_LIBRARIES = lib.a
lib_a_SOURCES = memchr-stub.c memchr.S memcmp-stub.c memcmp.S \
memcpy-stub.c memcpy.S memmove-stub.c memmove.S memset-stub.c \
- memset.S setjmp.S strchr-stub.c strchr.S strchrnul-stub.c \
- strchrnul.S strcmp-stub.c strcmp.S strcpy-stub.c strcpy.S \
- strlen-stub.c strlen.S strncmp-stub.c strncmp.S strnlen-stub.c \
- strnlen.S strrchr-stub.c strrchr.S
+ memset.S setjmp.S stpcpy-stub.c stpcpy.S strchr-stub.c \
+ strchr.S strchrnul-stub.c strchrnul.S strcmp-stub.c strcmp.S \
+ strcpy-stub.c strcpy.S strlen-stub.c strlen.S strncmp-stub.c \
+ strncmp.S strnlen-stub.c strnlen.S strrchr-stub.c strrchr.S
lib_a_CCASFLAGS = $(AM_CCASFLAGS)
lib_a_CFLAGS = $(AM_CFLAGS)
ACLOCAL_AMFLAGS = -I ../../.. -I ../../../..
@@ -312,6 +313,12 @@ lib_a-setjmp.o: setjmp.S
lib_a-setjmp.obj: setjmp.S
$(CCAS) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_a_CCASFLAGS) $(CCASFLAGS) -c -o lib_a-setjmp.obj `if test -f 'setjmp.S'; then $(CYGPATH_W) 'setjmp.S'; else $(CYGPATH_W) '$(srcdir)/setjmp.S'; fi`
+lib_a-stpcpy.o: stpcpy.S
+ $(CCAS) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_a_CCASFLAGS) $(CCASFLAGS) -c -o lib_a-stpcpy.o `test -f 'stpcpy.S' || echo '$(srcdir)/'`stpcpy.S
+
+lib_a-stpcpy.obj: stpcpy.S
+ $(CCAS) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_a_CCASFLAGS) $(CCASFLAGS) -c -o lib_a-stpcpy.obj `if test -f 'stpcpy.S'; then $(CYGPATH_W) 'stpcpy.S'; else $(CYGPATH_W) '$(srcdir)/stpcpy.S'; fi`
+
lib_a-strchr.o: strchr.S
$(CCAS) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_a_CCASFLAGS) $(CCASFLAGS) -c -o lib_a-strchr.o `test -f 'strchr.S' || echo '$(srcdir)/'`strchr.S
@@ -396,6 +403,12 @@ lib_a-memset-stub.o: memset-stub.c
lib_a-memset-stub.obj: memset-stub.c
$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_a_CFLAGS) $(CFLAGS) -c -o lib_a-memset-stub.obj `if test -f 'memset-stub.c'; then $(CYGPATH_W) 'memset-stub.c'; else $(CYGPATH_W) '$(srcdir)/memset-stub.c'; fi`
+lib_a-stpcpy-stub.o: stpcpy-stub.c
+ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_a_CFLAGS) $(CFLAGS) -c -o lib_a-stpcpy-stub.o `test -f 'stpcpy-stub.c' || echo '$(srcdir)/'`stpcpy-stub.c
+
+lib_a-stpcpy-stub.obj: stpcpy-stub.c
+ $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_a_CFLAGS) $(CFLAGS) -c -o lib_a-stpcpy-stub.obj `if test -f 'stpcpy-stub.c'; then $(CYGPATH_W) 'stpcpy-stub.c'; else $(CYGPATH_W) '$(srcdir)/stpcpy-stub.c'; fi`
+
lib_a-strchr-stub.o: strchr-stub.c
$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(lib_a_CFLAGS) $(CFLAGS) -c -o lib_a-strchr-stub.o `test -f 'strchr-stub.c' || echo '$(srcdir)/'`strchr-stub.c
Index: libc/machine/aarch64/stpcpy-stub.c
===================================================================
RCS file: libc/machine/aarch64/stpcpy-stub.c
diff -N libc/machine/aarch64/stpcpy-stub.c
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ libc/machine/aarch64/stpcpy-stub.c 6 Jan 2015 09:57:55 -0000 1.1
@@ -0,0 +1,31 @@
+/* Copyright (c) 2015, ARM Limited
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of the company nor the names of its contributors
+ may be used to endorse or promote products derived from this
+ software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */
+
+#if (defined (__OPTIMIZE_SIZE__) || defined (PREFER_SIZE_OVER_SPEED))
+# include "../../string/stpcpy.c"
+#else
+/* See stpcpy.S */
+#endif
Index: libc/machine/aarch64/stpcpy.S
===================================================================
RCS file: libc/machine/aarch64/stpcpy.S
diff -N libc/machine/aarch64/stpcpy.S
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ libc/machine/aarch64/stpcpy.S 6 Jan 2015 09:57:55 -0000 1.1
@@ -0,0 +1,34 @@
+/*
+ stpcpy - copy a string returning pointer to end.
+
+ Copyright (c) 2015 ARM Ltd.
+ All Rights Reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of the company nor the names of its contributors
+ may be used to endorse or promote products derived from this
+ software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */
+
+/* This is just a wrapper that uses strcpy code with appropriate
+ pre-defines. */
+
+#define BUILD_STPCPY
+#include "strcpy.S"
Index: libc/machine/aarch64/strcpy.S
===================================================================
RCS file: /cvs/src/src/newlib/libc/machine/aarch64/strcpy.S,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -p -r1.2 -r1.3
--- libc/machine/aarch64/strcpy.S 16 Dec 2014 15:48:58 -0000 1.2
+++ libc/machine/aarch64/strcpy.S 6 Jan 2015 09:57:55 -0000 1.3
@@ -1,7 +1,7 @@
/*
- strcpy - copy a string.
+ strcpy/stpcpy - copy a string returning pointer to start/end.
- Copyright (c) 2013, 2014 ARM Ltd.
+ Copyright (c) 2013, 2014, 2015 ARM Ltd.
All Rights Reserved.
Redistribution and use in source and binary forms, with or without
@@ -36,7 +36,9 @@
* ARMv8-a, AArch64, unaligned accesses, min page size 4k.
*/
-/* To test the page crossing code path more thoroughly, compile with
+/* To build as stpcpy, define BUILD_STPCPY before compiling this file.
+
+ To test the page crossing code path more thoroughly, compile with
-DSTRCPY_TEST_PAGE_CROSS - this will force all copies through the slower
entry path. This option is not intended for production use. */
@@ -64,6 +66,12 @@
#define len x16
#define to_align x17
+#ifdef BUILD_STPCPY
+#define STRCPY stpcpy
+#else
+#define STRCPY strcpy
+#endif
+
.macro def_fn f p2align=0
.text
.p2align \p2align
@@ -94,10 +102,16 @@
misaligned, crosses a page boundary - after that we move to aligned
fetches for the remainder of the string. */
+#ifdef STRCPY_TEST_PAGE_CROSS
+ /* Make everything that isn't Qword aligned look like a page cross. */
+#define MIN_PAGE_P2 4
+#else
#define MIN_PAGE_P2 12
+#endif
+
#define MIN_PAGE_SIZE (1 << MIN_PAGE_P2)
-def_fn strcpy p2align=6
+def_fn STRCPY p2align=6
/* For moderately short strings, the fastest way to do the copy is to
calculate the length of the string in the same way as strlen, then
essentially do a memcpy of the result. This avoids the need for
@@ -108,80 +122,112 @@ def_fn strcpy p2align=6
always be difficult - we mitigate against this by preferring
conditional select operations over branches whenever this is
feasible. */
- add tmp2, srcin, #15
+ and tmp2, srcin, #(MIN_PAGE_SIZE - 1)
mov zeroones, #REP8_01
and to_align, srcin, #15
- eor tmp2, tmp2, srcin
- mov dst, dstin
+ cmp tmp2, #(MIN_PAGE_SIZE - 16)
neg tmp1, to_align
-#ifdef STRCPY_TEST_PAGE_CROSS
- b .Lpage_cross
-#else
/* The first fetch will straddle a (possible) page boundary iff
srcin + 15 causes bit[MIN_PAGE_P2] to change value. A 16-byte
aligned string will never fail the page align check, so will
always take the fast path. */
- tbnz tmp2, #MIN_PAGE_P2, .Lpage_cross
-#endif
+ b.gt .Lpage_cross
+
+.Lpage_cross_ok:
ldp data1, data2, [srcin]
- add src, srcin, #16
+#ifdef __AARCH64EB__
+ /* Because we expect the end to be found within 16 characters
+ (profiling shows this is the most common case), it's worth
+ swapping the bytes now to save having to recalculate the
+ termination syndrome later. We preserve data1 and data2
+ so that we can re-use the values later on. */
+ rev tmp2, data1
+ sub tmp1, tmp2, zeroones
+ orr tmp2, tmp2, #REP8_7f
+ bics has_nul1, tmp1, tmp2
+ b.ne .Lfp_le8
+ rev tmp4, data2
+ sub tmp3, tmp4, zeroones
+ orr tmp4, tmp4, #REP8_7f
+#else
sub tmp1, data1, zeroones
orr tmp2, data1, #REP8_7f
+ bics has_nul1, tmp1, tmp2
+ b.ne .Lfp_le8
sub tmp3, data2, zeroones
orr tmp4, data2, #REP8_7f
- bic has_nul1, tmp1, tmp2
+#endif
bics has_nul2, tmp3, tmp4
- ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
- b.ne .Learly_end_found
- stp data1, data2, [dst], #16
- sub src, src, to_align
- sub dst, dst, to_align
- b .Lentry_no_page_cross
+ b.eq .Lbulk_entry
-.Lpage_cross:
- bic src, srcin, #15
- /* Start by loading two words at [srcin & ~15], then forcing the
- bytes that precede srcin to 0xff. This means they never look
- like termination bytes. */
- ldp data1, data2, [src], #16
- lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */
- tst to_align, #7
- csetm tmp2, ne
+ /* The string is short (<=16 bytes). We don't know exactly how
+ short though, yet. Work out the exact length so that we can
+ quickly select the optimal copy strategy. */
+.Lfp_gt8:
+ rev has_nul2, has_nul2
+ clz pos, has_nul2
+ mov tmp2, #56
+ add dst, dstin, pos, lsr #3 /* Bits to bytes. */
+ sub pos, tmp2, pos
#ifdef __AARCH64EB__
- lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
+ lsr data2, data2, pos
#else
- lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
+ lsl data2, data2, pos
#endif
- orr data1, data1, tmp2
- orr data2a, data2, tmp2
- cmp to_align, #8
- csinv data1, data1, xzr, lt
- csel data2, data2, data2a, lt
- sub tmp1, data1, zeroones
- orr tmp2, data1, #REP8_7f
- sub tmp3, data2, zeroones
- orr tmp4, data2, #REP8_7f
- bic has_nul1, tmp1, tmp2
- bics has_nul2, tmp3, tmp4
- ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
- b.ne .Learly_end_found
- ldp data1, data2, [src], #16
- sub tmp1, data1, zeroones
- orr tmp2, data1, #REP8_7f
- sub tmp3, data2, zeroones
- orr tmp4, data2, #REP8_7f
- bic has_nul1, tmp1, tmp2
- bics has_nul2, tmp3, tmp4
- ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
- b.ne .Learly_end_found
- /* We've now checked between 16 and 32 bytes, but not found a null,
- so we can safely start bulk copying. Start by refetching the
- first 16 bytes of the real string; we know this can't trap now. */
- ldp data1a, data2a, [srcin]
- stp data1a, data2a, [dst], #16
- sub dst, dst, to_align
- /* Everything is now set up, so we can just fall into the bulk
- copy loop. */
+ str data2, [dst, #1]
+ str data1, [dstin]
+#ifdef BUILD_STPCPY
+ add dstin, dst, #8
+#endif
+ ret
+
+.Lfp_le8:
+ rev has_nul1, has_nul1
+ clz pos, has_nul1
+ add dst, dstin, pos, lsr #3 /* Bits to bytes. */
+ subs tmp2, pos, #24 /* Pos in bits. */
+ b.lt .Lfp_lt4
+#ifdef __AARCH64EB__
+ mov tmp2, #56
+ sub pos, tmp2, pos
+ lsr data2, data1, pos
+ lsr data1, data1, #32
+#else
+ lsr data2, data1, tmp2
+#endif
+ /* 4->7 bytes to copy. */
+ str data2w, [dst, #-3]
+ str data1w, [dstin]
+#ifdef BUILD_STPCPY
+ mov dstin, dst
+#endif
+ ret
+.Lfp_lt4:
+ cbz pos, .Lfp_lt2
+ /* 2->3 bytes to copy. */
+#ifdef __AARCH64EB__
+ lsr data1, data1, #48
+#endif
+ strh data1w, [dstin]
+ /* Fall-through, one byte (max) to go. */
+.Lfp_lt2:
+ /* Null-terminated string. Last character must be zero! */
+ strb wzr, [dst]
+#ifdef BUILD_STPCPY
+ mov dstin, dst
+#endif
+ ret
+
+ .p2align 6
+ /* Aligning here ensures that the entry code and main loop all lies
+ within one 64-byte cache line. */
+.Lbulk_entry:
+ sub to_align, to_align, #16
+ stp data1, data2, [dstin]
+ sub src, srcin, to_align
+ sub dst, dstin, to_align
+ b .Lentry_no_page_cross
+
/* The inner loop deals with two Dwords at a time. This has a
slightly higher start-up cost, but we should win quite quickly,
especially on cores with a high number of issue slots per
@@ -225,72 +271,71 @@ def_fn strcpy p2align=6
add dst, dst, pos, lsr #3
ldp data1, data2, [src, #-32]
stp data1, data2, [dst, #-16]
+#ifdef BUILD_STPCPY
+ sub dstin, dst, #1
+#endif
ret
- /* The string is short (<32 bytes). We don't know exactly how
- short though, yet. Work out the exact length so that we can
- quickly select the optimal copy strategy. */
-.Learly_end_found:
- cmp has_nul1, #0
+.Lpage_cross:
+ bic src, srcin, #15
+ /* Start by loading two words at [srcin & ~15], then forcing the
+ bytes that precede srcin to 0xff. This means they never look
+ like termination bytes. */
+ ldp data1, data2, [src]
+ lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */
+ tst to_align, #7
+ csetm tmp2, ne
#ifdef __AARCH64EB__
- /* For big-endian, carry propagation (if the final byte in the
- string is 0x01) means we cannot use has_nul directly. The
- easiest way to get the correct byte is to byte-swap the data
- and calculate the syndrome a second time. */
- csel data1, data1, data2, ne
- rev data1, data1
+ lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
+#else
+ lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
+#endif
+ orr data1, data1, tmp2
+ orr data2a, data2, tmp2
+ cmp to_align, #8
+ csinv data1, data1, xzr, lt
+ csel data2, data2, data2a, lt
sub tmp1, data1, zeroones
orr tmp2, data1, #REP8_7f
+ sub tmp3, data2, zeroones
+ orr tmp4, data2, #REP8_7f
bic has_nul1, tmp1, tmp2
+ bics has_nul2, tmp3, tmp4
+ ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
+ b.eq .Lpage_cross_ok
+ /* We now need to make data1 and data2 look like they've been
+ loaded directly from srcin. Do a rotate on the 128-bit value. */
+ lsl tmp1, to_align, #3 /* Bytes->bits. */
+ neg tmp2, to_align, lsl #3
+#ifdef __AARCH64EB__
+ lsl data1a, data1, tmp1
+ lsr tmp4, data2, tmp2
+ lsl data2, data2, tmp1
+ orr tmp4, tmp4, data1a
+ cmp to_align, #8
+ csel data1, tmp4, data2, lt
+ rev tmp2, data1
+ rev tmp4, data2
+ sub tmp1, tmp2, zeroones
+ orr tmp2, tmp2, #REP8_7f
+ sub tmp3, tmp4, zeroones
+ orr tmp4, tmp4, #REP8_7f
#else
- csel has_nul1, has_nul1, has_nul2, ne
+ lsr data1a, data1, tmp1
+ lsl tmp4, data2, tmp2
+ lsr data2, data2, tmp1
+ orr tmp4, tmp4, data1a
+ cmp to_align, #8
+ csel data1, tmp4, data2, lt
+ sub tmp1, data1, zeroones
+ orr tmp2, data1, #REP8_7f
+ sub tmp3, data2, zeroones
+ orr tmp4, data2, #REP8_7f
#endif
- rev has_nul1, has_nul1
- sub tmp1, src, #7
- sub src, src, #15
- clz pos, has_nul1
- csel src, src, tmp1, ne
- sub dst, dstin, srcin
- add src, src, pos, lsr #3 /* Bits to bytes. */
- add dst, dst, src
- sub len, src, srcin
- cmp len, #8
- b.lt .Llt8
- cmp len, #16
- b.lt .Llt16
- /* 16->32 bytes to copy. */
- ldp data1, data2, [srcin]
- ldp data1a, data2a, [src, #-16]
- stp data1, data2, [dstin]
- stp data1a, data2a, [dst, #-16]
- ret
-.Llt16:
- /* 8->15 bytes to copy. */
- ldr data1, [srcin]
- ldr data2, [src, #-8]
- str data1, [dstin]
- str data2, [dst, #-8]
- ret
-.Llt8:
- cmp len, #4
- b.lt .Llt4
- /* 4->7 bytes to copy. */
- ldr data1w, [srcin]
- ldr data2w, [src, #-4]
- str data1w, [dstin]
- str data2w, [dst, #-4]
- ret
-.Llt4:
- cmp len, #2
- b.lt .Llt2
- /* 2->3 bytes to copy. */
- ldrh data1w, [srcin]
- strh data1w, [dstin]
- /* Fall-through, one byte (max) to go. */
-.Llt2:
- /* Null-terminated string. Last character must be zero! */
- strb wzr, [dst, #-1]
- ret
+ bic has_nul1, tmp1, tmp2
+ cbnz has_nul1, .Lfp_le8
+ bic has_nul2, tmp3, tmp4
+ b .Lfp_gt8
- .size strcpy, . - strcpy
+ .size STRCPY, . - STRCPY
#endif
More information about the Newlib
mailing list