/*
strcpy - copy a string.
- Copyright (c) 2013, 2014, ARM Limited
- All rights Reserved.
+ Copyright (c) 2013, 2014 ARM Ltd.
+ All Rights Reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
/* Assumptions:
*
- * ARMv8-a, AArch64, unaligned accesses
+ * ARMv8-a, AArch64, unaligned accesses, min page size 4k.
*/
+/* To test the page crossing code path more thoroughly, compile with
+ -DSTRCPY_TEST_PAGE_CROSS - this will force all copies through the slower
+ entry path. This option is not intended for production use. */
+
/* Arguments and results. */
#define dstin x0
-#define src x1
+#define srcin x1
/* Locals and temporaries. */
-#define dst x2
-#define data1 x3
-#define data1w w3
-#define data2 x4
-#define has_nul1 x5
-#define has_nul2 x6
-#define tmp1 x7
-#define tmp2 x8
-#define tmp3 x9
-#define tmp4 x10
-#define zeroones x11
+#define src x2
+#define dst x3
+#define data1 x4
+#define data1w w4
+#define data2 x5
+#define data2w w5
+#define has_nul1 x6
+#define has_nul2 x7
+#define tmp1 x8
+#define tmp2 x9
+#define tmp3 x10
+#define tmp4 x11
+#define zeroones x12
+#define data1a x13
+#define data2a x14
+#define pos x15
+#define len x16
+#define to_align x17
.macro def_fn f p2align=0
.text
\f:
.endm
+ /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
+ (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
+ can be done in parallel across the entire word. */
+
#define REP8_01 0x0101010101010101
#define REP8_7f 0x7f7f7f7f7f7f7f7f
#define REP8_80 0x8080808080808080
- /* Start of critial section -- keep to one 64Byte cache line. */
+ /* AArch64 systems have a minimum page size of 4k. We can do a quick
+ page size check for crossing this boundary on entry and if we
+ do not, then we can short-circuit much of the entry code. We
+ expect early page-crossing strings to be rare (probability of
+ 16/MIN_PAGE_SIZE ~= 0.4%), so the branch should be quite
+ predictable, even with random strings.
+
+ We don't bother checking for larger page sizes, the cost of setting
+ up the correct page size is just not worth the extra gain from
+ a small reduction in the cases taking the slow path. Note that
+ we only care about whether the first fetch, which may be
+ misaligned, crosses a page boundary - after that we move to aligned
+ fetches for the remainder of the string. */
+
+#define MIN_PAGE_P2 12
+#define MIN_PAGE_SIZE (1 << MIN_PAGE_P2)
+
def_fn strcpy p2align=6
+ /* For moderately short strings, the fastest way to do the copy is to
+ calculate the length of the string in the same way as strlen, then
+ essentially do a memcpy of the result. This avoids the need for
+ multiple byte copies and further means that by the time we
+ reach the bulk copy loop we know we can always use DWord
+ accesses. We expect strcpy to rarely be called repeatedly
+ with the same source string, so branch prediction is likely to
+ always be difficult - we mitigate against this by preferring
+ conditional select operations over branches whenever this is
+ feasible. */
+ add tmp2, srcin, #15
mov zeroones, #REP8_01
+ and to_align, srcin, #15
+ eor tmp2, tmp2, srcin
mov dst, dstin
- ands tmp1, src, #15
- b.ne .Lmisaligned
- /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
- (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
- can be done in parallel across the entire word. */
+ neg tmp1, to_align
+#ifdef STRCPY_TEST_PAGE_CROSS
+ b .Lpage_cross
+#else
+ /* The first fetch will straddle a (possible) page boundary iff
+ srcin + 15 causes bit[MIN_PAGE_P2] to change value. A 16-byte
+ aligned string will never fail the page align check, so will
+ always take the fast path. */
+ tbnz tmp2, #MIN_PAGE_P2, .Lpage_cross
+#endif
+ ldp data1, data2, [srcin]
+ add src, srcin, #16
+ sub tmp1, data1, zeroones
+ orr tmp2, data1, #REP8_7f
+ sub tmp3, data2, zeroones
+ orr tmp4, data2, #REP8_7f
+ bic has_nul1, tmp1, tmp2
+ bics has_nul2, tmp3, tmp4
+ ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
+ b.ne .Learly_end_found
+ stp data1, data2, [dst], #16
+ sub src, src, to_align
+ sub dst, dst, to_align
+ b .Lentry_no_page_cross
+
+.Lpage_cross:
+ bic src, srcin, #15
+ /* Start by loading two words at [srcin & ~15], then forcing the
+ bytes that precede srcin to 0xff. This means they never look
+ like termination bytes. */
+ ldp data1, data2, [src], #16
+ lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */
+ tst to_align, #7
+ csetm tmp2, ne
+#ifdef __AARCH64EB__
+ lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
+#else
+ lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
+#endif
+ orr data1, data1, tmp2
+ orr data2a, data2, tmp2
+ cmp to_align, #8
+ csinv data1, data1, xzr, lt
+ csel data2, data2, data2a, lt
+ sub tmp1, data1, zeroones
+ orr tmp2, data1, #REP8_7f
+ sub tmp3, data2, zeroones
+ orr tmp4, data2, #REP8_7f
+ bic has_nul1, tmp1, tmp2
+ bics has_nul2, tmp3, tmp4
+ ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
+ b.ne .Learly_end_found
+ ldp data1, data2, [src], #16
+ sub tmp1, data1, zeroones
+ orr tmp2, data1, #REP8_7f
+ sub tmp3, data2, zeroones
+ orr tmp4, data2, #REP8_7f
+ bic has_nul1, tmp1, tmp2
+ bics has_nul2, tmp3, tmp4
+ ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
+ b.ne .Learly_end_found
+ /* We've now checked between 16 and 32 bytes, but not found a null,
+ so we can safely start bulk copying. Start by refetching the
+ first 16 bytes of the real string; we know this can't trap now. */
+ ldp data1a, data2a, [srcin]
+ stp data1a, data2a, [dst], #16
+ sub dst, dst, to_align
+ /* Everything is now set up, so we can just fall into the bulk
+ copy loop. */
/* The inner loop deals with two Dwords at a time. This has a
slightly higher start-up cost, but we should win quite quickly,
especially on cores with a high number of issue slots per
cycle, as we get much better parallelism out of the operations. */
- b .Lfirst_pass
.Lmain_loop:
stp data1, data2, [dst], #16
-.Lstartloop_fast:
+.Lentry_no_page_cross:
ldp data1, data2, [src], #16
sub tmp1, data1, zeroones
orr tmp2, data1, #REP8_7f
bics has_nul2, tmp3, tmp4
ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
b.eq .Lmain_loop
- /* End of critical section -- keep to one 64Byte cache line. */
-
- cbnz has_nul1, .Lnul_in_data1_fast
-.Lnul_in_data2_fast:
- str data1, [dst], #8
-.Lnul_in_data2_fast_after_d1:
- /* For a NUL in data2, we always know that we've moved at least 8
- bytes, so no need for a slow path. */
-#ifdef __AARCH64EB__
- /* For big-endian only, carry propagation means we can't trust
- the MSB of the syndrome value calculated above (the byte
- sequence 01 00 will generate a syndrome of 80 80 rather than
- 00 80). We get around this by byte-swapping the data and
- re-calculating. */
- rev data2, data2
- sub tmp1, data2, zeroones
- orr tmp2, data2, #REP8_7f
- bic has_nul2, tmp1, tmp2
-#endif
- rev has_nul2, has_nul2
- sub src, src, #(8+7)
- clz has_nul2, has_nul2
- lsr has_nul2, has_nul2, #3 /* Bits to bytes. */
- sub dst, dst, #7
- ldr data2, [src, has_nul2]
- str data2, [dst, has_nul2]
- ret
-.Lnul_in_data1_fast:
- /* Since we know we've already copied at least 8 bytes, we can
- safely handle the tail with one misaligned dword move. To do this
- we calculate the location of the trailing NUL byte and go seven
- bytes back from that. */
+ /* Since we know we are copying at least 16 bytes, the fastest way
+ to deal with the tail is to determine the location of the
+ trailing NUL, then (re)copy the 16 bytes leading up to that. */
+ cmp has_nul1, #0
#ifdef __AARCH64EB__
- /* For big-endian only, carry propagation means we can't trust
- the MSB of the syndrome value calculated above (the byte
- sequence 01 00 will generate a syndrome of 80 80 rather than
- 00 80). We get around this by byte-swapping the data and
- re-calculating. */
+ /* For big-endian, carry propagation (if the final byte in the
+ string is 0x01) means we cannot use has_nul directly. The
+ easiest way to get the correct byte is to byte-swap the data
+ and calculate the syndrome a second time. */
+ csel data1, data1, data2, ne
rev data1, data1
sub tmp1, data1, zeroones
orr tmp2, data1, #REP8_7f
bic has_nul1, tmp1, tmp2
+#else
+ csel has_nul1, has_nul1, has_nul2, ne
#endif
rev has_nul1, has_nul1
- sub src, src, #(16+7)
- clz has_nul1, has_nul1
- lsr has_nul1, has_nul1, #3 /* Bits to bytes. */
- sub dst, dst, #7
- ldr data1, [src, has_nul1]
- str data1, [dst, has_nul1]
+ clz pos, has_nul1
+ add tmp1, pos, #72
+ add pos, pos, #8
+ csel pos, pos, tmp1, ne
+ add src, src, pos, lsr #3
+ add dst, dst, pos, lsr #3
+ ldp data1, data2, [src, #-32]
+ stp data1, data2, [dst, #-16]
ret
-.Lfirst_pass:
- ldp data1, data2, [src], #16
+ /* The string is short (<32 bytes). We don't know exactly how
+ short though, yet. Work out the exact length so that we can
+ quickly select the optimal copy strategy. */
+.Learly_end_found:
+ cmp has_nul1, #0
+#ifdef __AARCH64EB__
+ /* For big-endian, carry propagation (if the final byte in the
+ string is 0x01) means we cannot use has_nul directly. The
+ easiest way to get the correct byte is to byte-swap the data
+ and calculate the syndrome a second time. */
+ csel data1, data1, data2, ne
+ rev data1, data1
sub tmp1, data1, zeroones
orr tmp2, data1, #REP8_7f
- sub tmp3, data2, zeroones
- orr tmp4, data2, #REP8_7f
bic has_nul1, tmp1, tmp2
- bics has_nul2, tmp3, tmp4
- ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
- b.eq .Lmain_loop
-
- cbz has_nul1, .Lnul_in_data2_fast
-.Lnul_in_data1:
- /* Slow path. We can't be sure we've moved at least 8 bytes, so
- fall back to a slow byte-by byte store of the bits already
- loaded.
-
- The worst case when coming through this path is that we've had
- to copy seven individual bytes to get to alignment and we then
- have to copy another seven (eight for big-endian) again here.
- We could try to detect that case (and any case where more than
- eight bytes have to be copied), but it really doesn't seem
- worth it. */
-#ifdef __AARCH64EB__
- rev data1, data1
#else
- /* On little-endian, we can easily check if the NULL byte was
- in the last byte of the Dword. For big-endian we'd have to
- recalculate the syndrome, which is unlikely to be worth it. */
- lsl has_nul1, has_nul1, #8
- cbnz has_nul1, 1f
- str data1, [dst]
- ret
+ csel has_nul1, has_nul1, has_nul2, ne
#endif
-1:
- strb data1w, [dst], #1
- tst data1, #0xff
- lsr data1, data1, #8
- b.ne 1b
-.Ldone:
+ rev has_nul1, has_nul1
+ sub tmp1, src, #7
+ sub src, src, #15
+ clz pos, has_nul1
+ csel src, src, tmp1, ne
+ sub dst, dstin, srcin
+ add src, src, pos, lsr #3 /* Bits to bytes. */
+ add dst, dst, src
+ sub len, src, srcin
+ cmp len, #8
+ b.lt .Llt8
+ cmp len, #16
+ b.lt .Llt16
+ /* 16->32 bytes to copy. */
+ ldp data1, data2, [srcin]
+ ldp data1a, data2a, [src, #-16]
+ stp data1, data2, [dstin]
+ stp data1a, data2a, [dst, #-16]
+ ret
+.Llt16:
+ /* 8->15 bytes to copy. */
+ ldr data1, [srcin]
+ ldr data2, [src, #-8]
+ str data1, [dstin]
+ str data2, [dst, #-8]
+ ret
+.Llt8:
+ cmp len, #4
+ b.lt .Llt4
+ /* 4->7 bytes to copy. */
+ ldr data1w, [srcin]
+ ldr data2w, [src, #-4]
+ str data1w, [dstin]
+ str data2w, [dst, #-4]
+ ret
+.Llt4:
+ cmp len, #2
+ b.lt .Llt2
+ /* 2->3 bytes to copy. */
+ ldrh data1w, [srcin]
+ strh data1w, [dstin]
+ /* Fall-through, one byte (max) to go. */
+.Llt2:
+ /* Null-terminated string. Last character must be zero! */
+ strb wzr, [dst, #-1]
ret
-
-.Lmisaligned:
- cmp tmp1, #8
- b.ge 2f
- /* There's at least one Dword before we reach alignment, so we can
- deal with that efficiently. */
- ldr data1, [src]
- bic src, src, #15
- sub tmp3, data1, zeroones
- orr tmp4, data1, #REP8_7f
- bics has_nul1, tmp3, tmp4
- b.ne .Lnul_in_data1
- str data1, [dst], #8
- ldr data2, [src, #8]
- add src, src, #16
- sub dst, dst, tmp1
- sub tmp3, data2, zeroones
- orr tmp4, data2, #REP8_7f
- bics has_nul2, tmp3, tmp4
- b.ne .Lnul_in_data2_fast_after_d1
- str data2, [dst], #8
- /* We can by-pass the first-pass version of the loop in this case
- since we know that at least 8 bytes have already been copied. */
- b .Lstartloop_fast
-
-2:
- sub tmp1, tmp1, #16
-3:
- ldrb data1w, [src], #1
- strb data1w, [dst], #1
- cbz data1w, .Ldone
- add tmp1, tmp1, #1
- cbnz tmp1, 3b
- b .Lfirst_pass
.size strcpy, . - strcpy
#endif
git://sourceware.org / newlib-cygwin.git / commitdiff