[PATCH][AArch64] Tune memcpy
Wilco Dijkstra
Wilco.Dijkstra@arm.com
Fri Nov 6 13:29:00 GMT 2015
This patch further tunes memcpy on AArch64 - avoid one branch for sizes 1-3,
add a prefetch and improve small copies that are exact powers of 2.
OK for commit?
ChangeLog:
2015-11-06 Wilco Dijkstra <wdijkstr@arm.com>
* newlib/libc/machine/aarch64/memcpy.S (memcpy):
Further tuning for performance.
---
newlib/libc/machine/aarch64/memcpy.S | 53
++++++++++++++++++++----------------
1 file changed, 29 insertions(+), 24 deletions(-)
diff --git a/newlib/libc/machine/aarch64/memcpy.S
b/newlib/libc/machine/aarch64/memcpy.S
index c109684..b53d125 100644
--- a/newlib/libc/machine/aarch64/memcpy.S
+++ b/newlib/libc/machine/aarch64/memcpy.S
@@ -73,6 +73,7 @@
#define A_h x7
#define A_hw w7
#define B_l x8
+#define B_lw w8
#define B_h x9
#define C_l x10
#define C_h x11
@@ -104,21 +105,40 @@
*/
def_fn memcpy p2align=6
+ prfm PLDL1KEEP, [src]
add srcend, src, count
add dstend, dstin, count
+ cmp count, 16
+ b.ls L(copy16)
cmp count, 96
b.hi L(copy_long)
- cmp count, 16
- b.hs L(copy_medium)
+ /* Medium copies: 17..96 bytes. */
+ sub tmp1, count, 1
+ ldp A_l, A_h, [src]
+ tbnz tmp1, 6, L(copy96)
+ ldp D_l, D_h, [srcend, -16]
+ tbz tmp1, 5, 1f
+ ldp B_l, B_h, [src, 16]
+ ldp C_l, C_h, [srcend, -32]
+ stp B_l, B_h, [dstin, 16]
+ stp C_l, C_h, [dstend, -32]
+1:
+ stp A_l, A_h, [dstin]
+ stp D_l, D_h, [dstend, -16]
+ ret
+
+ .p2align 4
/* Small copies: 0..16 bytes. */
L(copy16):
- tbz count, 3, 1f
+ cmp count, 8
+ b.lo 1f
ldr A_l, [src]
ldr A_h, [srcend, -8]
str A_l, [dstin]
str A_h, [dstend, -8]
ret
+ .p2align 4
1:
tbz count, 2, 1f
ldr A_lw, [src]
@@ -126,33 +146,18 @@ L(copy16):
str A_lw, [dstin]
str A_hw, [dstend, -4]
ret
- .p2align 4
1:
cbz count, 2f
+ lsr tmp1, count, 1
ldrb A_lw, [src]
- tbz count, 1, 1f
- ldrh A_hw, [srcend, -2]
- strh A_hw, [dstend, -2]
-1: strb A_lw, [dstin]
+ ldrb A_hw, [srcend, -1]
+ ldrb B_lw, [src, tmp1]
+ strb A_lw, [dstin]
+ strb B_lw, [dstin, tmp1]
+ strb A_hw, [dstend, -1]
2: ret
.p2align 4
- /* Medium copies: 17..96 bytes. */
-L(copy_medium):
- ldp A_l, A_h, [src]
- tbnz count, 6, L(copy96)
- ldp D_l, D_h, [srcend, -16]
- tbz count, 5, 1f
- ldp B_l, B_h, [src, 16]
- ldp C_l, C_h, [srcend, -32]
- stp B_l, B_h, [dstin, 16]
- stp C_l, C_h, [dstend, -32]
-1:
- stp A_l, A_h, [dstin]
- stp D_l, D_h, [dstend, -16]
- ret
-
- .p2align 4
/* Copy 64..96 bytes. Copy 64 bytes from the start and
32 bytes from the end. */
L(copy96):
--
1.9.1
-------------- next part --------------
A non-text attachment was scrubbed...
Name: Tune-memcpy.patch
Type: application/octet-stream
Size: 2157 bytes
Desc: not available
URL: <http://sourceware.org/pipermail/newlib/attachments/20151106/0418172b/attachment.obj>
More information about the Newlib
mailing list