[PATCH] Save and restore xmm0-xmm7 in _dl_runtime_resolve
Ondřej Bílka
neleai@seznam.cz
Sun Jul 26 13:16:00 GMT 2015
On Sat, Jul 11, 2015 at 04:50:02PM -0700, H.J. Lu wrote:
> On Sat, Jul 11, 2015 at 01:27:42PM -0700, H.J. Lu wrote:
> > On Sat, Jul 11, 2015 at 12:46:54PM +0200, OndÅ™ej BÃlka wrote:
> > > On Thu, Jul 09, 2015 at 09:07:24AM -0700, H.J. Lu wrote:
> > > > On Thu, Jul 9, 2015 at 7:28 AM, OndÅ™ej BÃlka <neleai@seznam.cz> wrote:
> > > > > On Thu, Jul 09, 2015 at 07:12:24AM -0700, H.J. Lu wrote:
> > > > >> On Thu, Jul 9, 2015 at 6:37 AM, Zamyatin, Igor <igor.zamyatin@intel.com> wrote:
> > > > >> >> On Wed, Jul 8, 2015 at 8:56 AM, Zamyatin, Igor <igor.zamyatin@intel.com>
> > > > >> >> wrote:
> > > > >> >> > Fixed in the attached patch
> > > > >> >> >
> > > > >> >>
> > > > >> >> I fixed some typos and updated sysdeps/i386/configure for
> > > > >> >> HAVE_MPX_SUPPORT. Please verify both with HAVE_MPX_SUPPORT and
> > > > >> >> without on i386 and x86-64.
> > > > >> >
> > > > >> > Done, all works fine
> > > > >> >
> > > > >>
> > > > >> I checked it in for you.
> > > > >>
> > > > > These are nice but you could have same problem with lazy tls allocation.
> > > > > I wrote patch to merge trampolines, which now conflicts. Could you write
> > > > > similar patch to solve that? Original purpose was to always save xmm
> > > > > registers so we could use sse2 routines which speeds up lookup time.
> > > >
> > > > So we will preserve only xmm0 to xmm7 in _dl_runtime_resolve? How
> > > > much gain it will give us?
> > > >
> > > I couldn't measure that without patch. Gain now would be big as we now
> > > use byte-by-byte loop to check symbol name which is slow, especially
> > > with c++ name mangling. Would be following benchmark good to measure
> > > speedup or do I need to measure startup time which is bit harder?
> > >
> >
> > Please try this.
> >
>
> We have to use movups instead of movaps due to
>
> https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066
>
>
Thanks, this looks promising.
I think how to do definite benchmark, Now I have evidence that its
likely improvement but not definite.
I found that benchmark that i intended causes too much noise and I
didn't get useful from that yet. It was creating 1000 functions in
library and calling them from main where performance between runs vary
by factor of 3 for same implementation.
I have indirect evidence. With attached patch to use sse2 routines I
decreased startup time of running binaries when you run "make bench"
by ~6000 cycles and dlopen time by 4% on haswell and ivy bridge.
See results on haswell of
LD_DEBUG=statistics make bench &> old_rtld
that are large so you could browse these here
http://kam.mff.cuni.cz/~ondra/old_rtld
http://kam.mff.cuni.cz/~ondra/new_rtld
For dlopen benchmark I measure ten times performance of
dlopen(RTLD_DEFAULT,"memcpy");
dlopen(RTLD_DEFAULT,"strlen");
Without patch I get
624.49 559.58 556.6 556.04 558.42 557.86 559.46 555.17 556.93 555.32
and with patch
604.71 536.74 536.08 535.78 534.11 533.67 534.8 534.8 533.46 536.08
I attached vip patches, I didn't change memcpy yet.
So if you have idea how directly measure fixup change it would be
welcome.
-------------- next part --------------
>From 8bad3c9751cba151b5f4ad2108bf2b860705c6eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20B=C3=ADlka?= <neleai@seznam.cz>
Date: Sun, 26 Jul 2015 10:15:56 +0200
Subject: [PATCH 2/4] dlopen benchmark
---
benchtests/Makefile | 3 ++-
benchtests/bench-dlopen.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 49 insertions(+), 1 deletion(-)
create mode 100644 benchtests/bench-dlopen.c
diff --git a/benchtests/Makefile b/benchtests/Makefile
index 8e615e5..9e82e43 100644
--- a/benchtests/Makefile
+++ b/benchtests/Makefile
@@ -30,7 +30,7 @@ bench-pthread := pthread_once
bench := $(bench-math) $(bench-pthread)
# String function benchmarks.
-string-bench := bcopy bzero memccpy memchr memcmp memcpy memmem memmove \
+string-bench := dlopen bcopy bzero memccpy memchr memcmp memcpy memmem memmove \
mempcpy memset rawmemchr stpcpy stpncpy strcasecmp strcasestr \
strcat strchr strchrnul strcmp strcpy strcspn strlen \
strncasecmp strncat strncmp strncpy strnlen strpbrk strrchr \
@@ -57,6 +57,7 @@ CFLAGS-bench-ffsll.c += -fno-builtin
bench-malloc := malloc-thread
+$(objpfx)bench-dlopen: -ldl
$(addprefix $(objpfx)bench-,$(bench-math)): $(libm)
$(addprefix $(objpfx)bench-,$(bench-pthread)): $(shared-thread-library)
$(objpfx)bench-malloc-thread: $(shared-thread-library)
diff --git a/benchtests/bench-dlopen.c b/benchtests/bench-dlopen.c
new file mode 100644
index 0000000..b47d18b
--- /dev/null
+++ b/benchtests/bench-dlopen.c
@@ -0,0 +1,47 @@
+/* Measure strcpy functions.
+ Copyright (C) 2013-2015 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+#include <dlfcn.h>
+#include <string.h>
+#include "bench-timing.h"
+
+int
+main (void)
+{
+ long ret = 0;
+
+ size_t i, j,iters = 100;
+ timing_t start, stop, cur;
+ for (j=0;j<10;j++)
+ {
+ TIMING_NOW (start);
+ for (i = 0; i < iters; ++i)
+ {
+ ret += (long) dlsym (RTLD_DEFAULT, "strlen");
+ ret += (long) dlsym (RTLD_DEFAULT, "memcpy");
+ }
+ TIMING_NOW (stop);
+
+ TIMING_DIFF (cur, start, stop);
+
+ TIMING_PRINT_MEAN ((double) cur, (double) iters);
+ }
+
+ return ret;
+}
+
+
--
2.1.4
-------------- next part --------------
>From e4ca50b9d4ba9cd378a740623f2da889afbe6309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20B=C3=ADlka?= <neleai@seznam.cz>
Date: Sun, 26 Jul 2015 10:39:15 +0200
Subject: [PATCH 4/4] rtld
---
sysdeps/x86_64/multiarch/rtld-memcmp.c | 1 -
sysdeps/x86_64/multiarch/rtld-memset.S | 1 -
sysdeps/x86_64/rtld-memcmp.c | 1 -
sysdeps/x86_64/rtld-memset.S | 37 -----
sysdeps/x86_64/rtld-strchr.S | 288 ---------------------------------
sysdeps/x86_64/rtld-strlen.S | 136 ----------------
sysdeps/x86_64/strcmp.S | 1 +
7 files changed, 1 insertion(+), 464 deletions(-)
delete mode 100644 sysdeps/x86_64/multiarch/rtld-memcmp.c
delete mode 100644 sysdeps/x86_64/multiarch/rtld-memset.S
delete mode 100644 sysdeps/x86_64/rtld-memcmp.c
delete mode 100644 sysdeps/x86_64/rtld-memset.S
delete mode 100644 sysdeps/x86_64/rtld-strchr.S
delete mode 100644 sysdeps/x86_64/rtld-strlen.S
diff --git a/sysdeps/x86_64/multiarch/rtld-memcmp.c b/sysdeps/x86_64/multiarch/rtld-memcmp.c
deleted file mode 100644
index 0f27135..0000000
--- a/sysdeps/x86_64/multiarch/rtld-memcmp.c
+++ /dev/null
@@ -1 +0,0 @@
-#include "../rtld-memcmp.c"
diff --git a/sysdeps/x86_64/multiarch/rtld-memset.S b/sysdeps/x86_64/multiarch/rtld-memset.S
deleted file mode 100644
index 8092aa0..0000000
--- a/sysdeps/x86_64/multiarch/rtld-memset.S
+++ /dev/null
@@ -1 +0,0 @@
-#include "../rtld-memset.S"
diff --git a/sysdeps/x86_64/rtld-memcmp.c b/sysdeps/x86_64/rtld-memcmp.c
deleted file mode 100644
index 2ee4032..0000000
--- a/sysdeps/x86_64/rtld-memcmp.c
+++ /dev/null
@@ -1 +0,0 @@
-#include <string/memcmp.c>
diff --git a/sysdeps/x86_64/rtld-memset.S b/sysdeps/x86_64/rtld-memset.S
deleted file mode 100644
index f8df333..0000000
--- a/sysdeps/x86_64/rtld-memset.S
+++ /dev/null
@@ -1,37 +0,0 @@
-/* memset implementation for the dynamic linker. This is separate from the
- libc implementation to avoid writing to SSE registers.
- Copyright (C) 2013-2015 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#include <sysdep.h>
-#include "asm-syntax.h"
-
-
- .text
-/* void *memset (void *dest, char c, size_t count)
- dest => %rdi
- c => %rsi
- count => %rdx */
-ENTRY (memset)
- mov %rdx, %rcx
- movzbl %sil, %eax
- mov %rdi, %rdx
- rep stosb
- mov %rdx, %rax
- ret
-END (memset)
-libc_hidden_builtin_def (memset)
diff --git a/sysdeps/x86_64/rtld-strchr.S b/sysdeps/x86_64/rtld-strchr.S
deleted file mode 100644
index cc694d7..0000000
--- a/sysdeps/x86_64/rtld-strchr.S
+++ /dev/null
@@ -1,288 +0,0 @@
-/* strchr (str, ch) -- Return pointer to first occurrence of CH in STR.
- For AMD x86-64.
- Copyright (C) 2002-2015 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#include <sysdep.h>
-#include "asm-syntax.h"
-
-
- .text
-ENTRY (strchr)
-
- /* Before we start with the main loop we process single bytes
- until the source pointer is aligned. This has two reasons:
- 1. aligned 64-bit memory access is faster
- and (more important)
- 2. we process in the main loop 64 bit in one step although
- we don't know the end of the string. But accessing at
- 8-byte alignment guarantees that we never access illegal
- memory if this would not also be done by the trivial
- implementation (this is because all processor inherent
- boundaries are multiples of 8). */
-
- movq %rdi, %rdx
- andl $7, %edx /* Mask alignment bits */
- movq %rdi, %rax /* duplicate destination. */
- jz 1f /* aligned => start loop */
- neg %edx
- addl $8, %edx /* Align to 8 bytes. */
-
- /* Search the first bytes directly. */
-0: movb (%rax), %cl /* load byte */
- cmpb %cl,%sil /* compare byte. */
- je 6f /* target found */
- testb %cl,%cl /* is byte NUL? */
- je 7f /* yes => return NULL */
- incq %rax /* increment pointer */
- decl %edx
- jnz 0b
-
-
-1:
- /* At the moment %rsi contains C. What we need for the
- algorithm is C in all bytes of the register. Avoid
- operations on 16 bit words because these require an
- prefix byte (and one more cycle). */
- /* Populate 8 bit data to full 64-bit. */
- movabs $0x0101010101010101,%r9
- movzbl %sil,%edx
- imul %rdx,%r9
-
- movq $0xfefefefefefefeff, %r8 /* Save magic. */
-
- /* We exit the loop if adding MAGIC_BITS to LONGWORD fails to
- change any of the hole bits of LONGWORD.
-
- 1) Is this safe? Will it catch all the zero bytes?
- Suppose there is a byte with all zeros. Any carry bits
- propagating from its left will fall into the hole at its
- least significant bit and stop. Since there will be no
- carry from its most significant bit, the LSB of the
- byte to the left will be unchanged, and the zero will be
- detected.
-
- 2) Is this worthwhile? Will it ignore everything except
- zero bytes? Suppose every byte of QUARDWORD has a bit set
- somewhere. There will be a carry into bit 8. If bit 8
- is set, this will carry into bit 16. If bit 8 is clear,
- one of bits 9-15 must be set, so there will be a carry
- into bit 16. Similarly, there will be a carry into bit
- 24 tec.. If one of bits 54-63 is set, there will be a carry
- into bit 64 (=carry flag), so all of the hole bits will
- be changed.
-
- 3) But wait! Aren't we looking for C, not zero?
- Good point. So what we do is XOR LONGWORD with a longword,
- each of whose bytes is C. This turns each byte that is C
- into a zero. */
-
- .p2align 4
-4:
- /* Main Loop is unrolled 4 times. */
- /* First unroll. */
- movq (%rax), %rcx /* get double word (= 8 bytes) in question */
- addq $8,%rax /* adjust pointer for next word */
- movq %r8, %rdx /* magic value */
- xorq %r9, %rcx /* XOR with qword c|...|c => bytes of str == c
- are now 0 */
- addq %rcx, %rdx /* add the magic value to the word. We get
- carry bits reported for each byte which
- is *not* 0 */
- jnc 3f /* highest byte is NUL => return pointer */
- xorq %rcx, %rdx /* (word+magic)^word */
- orq %r8, %rdx /* set all non-carry bits */
- incq %rdx /* add 1: if one carry bit was *not* set
- the addition will not result in 0. */
- jnz 3f /* found c => return pointer */
-
- /* The quadword we looked at does not contain the value we're looking
- for. Let's search now whether we have reached the end of the
- string. */
- xorq %r9, %rcx /* restore original dword without reload */
- movq %r8, %rdx /* magic value */
- addq %rcx, %rdx /* add the magic value to the word. We get
- carry bits reported for each byte which
- is *not* 0 */
- jnc 7f /* highest byte is NUL => return NULL */
- xorq %rcx, %rdx /* (word+magic)^word */
- orq %r8, %rdx /* set all non-carry bits */
- incq %rdx /* add 1: if one carry bit was *not* set
- the addition will not result in 0. */
- jnz 7f /* found NUL => return NULL */
-
- /* Second unroll. */
- movq (%rax), %rcx /* get double word (= 8 bytes) in question */
- addq $8,%rax /* adjust pointer for next word */
- movq %r8, %rdx /* magic value */
- xorq %r9, %rcx /* XOR with qword c|...|c => bytes of str == c
- are now 0 */
- addq %rcx, %rdx /* add the magic value to the word. We get
- carry bits reported for each byte which
- is *not* 0 */
- jnc 3f /* highest byte is NUL => return pointer */
- xorq %rcx, %rdx /* (word+magic)^word */
- orq %r8, %rdx /* set all non-carry bits */
- incq %rdx /* add 1: if one carry bit was *not* set
- the addition will not result in 0. */
- jnz 3f /* found c => return pointer */
-
- /* The quadword we looked at does not contain the value we're looking
- for. Let's search now whether we have reached the end of the
- string. */
- xorq %r9, %rcx /* restore original dword without reload */
- movq %r8, %rdx /* magic value */
- addq %rcx, %rdx /* add the magic value to the word. We get
- carry bits reported for each byte which
- is *not* 0 */
- jnc 7f /* highest byte is NUL => return NULL */
- xorq %rcx, %rdx /* (word+magic)^word */
- orq %r8, %rdx /* set all non-carry bits */
- incq %rdx /* add 1: if one carry bit was *not* set
- the addition will not result in 0. */
- jnz 7f /* found NUL => return NULL */
- /* Third unroll. */
- movq (%rax), %rcx /* get double word (= 8 bytes) in question */
- addq $8,%rax /* adjust pointer for next word */
- movq %r8, %rdx /* magic value */
- xorq %r9, %rcx /* XOR with qword c|...|c => bytes of str == c
- are now 0 */
- addq %rcx, %rdx /* add the magic value to the word. We get
- carry bits reported for each byte which
- is *not* 0 */
- jnc 3f /* highest byte is NUL => return pointer */
- xorq %rcx, %rdx /* (word+magic)^word */
- orq %r8, %rdx /* set all non-carry bits */
- incq %rdx /* add 1: if one carry bit was *not* set
- the addition will not result in 0. */
- jnz 3f /* found c => return pointer */
-
- /* The quadword we looked at does not contain the value we're looking
- for. Let's search now whether we have reached the end of the
- string. */
- xorq %r9, %rcx /* restore original dword without reload */
- movq %r8, %rdx /* magic value */
- addq %rcx, %rdx /* add the magic value to the word. We get
- carry bits reported for each byte which
- is *not* 0 */
- jnc 7f /* highest byte is NUL => return NULL */
- xorq %rcx, %rdx /* (word+magic)^word */
- orq %r8, %rdx /* set all non-carry bits */
- incq %rdx /* add 1: if one carry bit was *not* set
- the addition will not result in 0. */
- jnz 7f /* found NUL => return NULL */
- /* Fourth unroll. */
- movq (%rax), %rcx /* get double word (= 8 bytes) in question */
- addq $8,%rax /* adjust pointer for next word */
- movq %r8, %rdx /* magic value */
- xorq %r9, %rcx /* XOR with qword c|...|c => bytes of str == c
- are now 0 */
- addq %rcx, %rdx /* add the magic value to the word. We get
- carry bits reported for each byte which
- is *not* 0 */
- jnc 3f /* highest byte is NUL => return pointer */
- xorq %rcx, %rdx /* (word+magic)^word */
- orq %r8, %rdx /* set all non-carry bits */
- incq %rdx /* add 1: if one carry bit was *not* set
- the addition will not result in 0. */
- jnz 3f /* found c => return pointer */
-
- /* The quadword we looked at does not contain the value we're looking
- for. Let's search now whether we have reached the end of the
- string. */
- xorq %r9, %rcx /* restore original dword without reload */
- movq %r8, %rdx /* magic value */
- addq %rcx, %rdx /* add the magic value to the word. We get
- carry bits reported for each byte which
- is *not* 0 */
- jnc 7f /* highest byte is NUL => return NULL */
- xorq %rcx, %rdx /* (word+magic)^word */
- orq %r8, %rdx /* set all non-carry bits */
- incq %rdx /* add 1: if one carry bit was *not* set
- the addition will not result in 0. */
- jz 4b /* no NUL found => restart loop */
-
-
-7: /* Return NULL. */
- xorl %eax, %eax
- retq
-
-
- /* We now scan for the byte in which the character was matched.
- But we have to take care of the case that a NUL char is
- found before this in the dword. Note that we XORed %rcx
- with the byte we're looking for, therefore the tests below look
- reversed. */
-
-
- .p2align 4 /* Align, it's a jump target. */
-3: movq %r9,%rdx /* move to %rdx so that we can access bytes */
- subq $8,%rax /* correct pointer increment. */
- testb %cl, %cl /* is first byte C? */
- jz 6f /* yes => return pointer */
- cmpb %dl, %cl /* is first byte NUL? */
- je 7b /* yes => return NULL */
- incq %rax /* increment pointer */
-
- testb %ch, %ch /* is second byte C? */
- jz 6f /* yes => return pointer */
- cmpb %dl, %ch /* is second byte NUL? */
- je 7b /* yes => return NULL? */
- incq %rax /* increment pointer */
-
- shrq $16, %rcx /* make upper bytes accessible */
- testb %cl, %cl /* is third byte C? */
- jz 6f /* yes => return pointer */
- cmpb %dl, %cl /* is third byte NUL? */
- je 7b /* yes => return NULL */
- incq %rax /* increment pointer */
-
- testb %ch, %ch /* is fourth byte C? */
- jz 6f /* yes => return pointer */
- cmpb %dl, %ch /* is fourth byte NUL? */
- je 7b /* yes => return NULL? */
- incq %rax /* increment pointer */
-
- shrq $16, %rcx /* make upper bytes accessible */
- testb %cl, %cl /* is fifth byte C? */
- jz 6f /* yes => return pointer */
- cmpb %dl, %cl /* is fifth byte NUL? */
- je 7b /* yes => return NULL */
- incq %rax /* increment pointer */
-
- testb %ch, %ch /* is sixth byte C? */
- jz 6f /* yes => return pointer */
- cmpb %dl, %ch /* is sixth byte NUL? */
- je 7b /* yes => return NULL? */
- incq %rax /* increment pointer */
-
- shrq $16, %rcx /* make upper bytes accessible */
- testb %cl, %cl /* is seventh byte C? */
- jz 6f /* yes => return pointer */
- cmpb %dl, %cl /* is seventh byte NUL? */
- je 7b /* yes => return NULL */
-
- /* It must be in the eigth byte and it cannot be NUL. */
- incq %rax
-
-6:
- nop
- retq
-END (strchr)
-
-weak_alias (strchr, index)
-libc_hidden_builtin_def (strchr)
diff --git a/sysdeps/x86_64/rtld-strlen.S b/sysdeps/x86_64/rtld-strlen.S
deleted file mode 100644
index 1328652..0000000
--- a/sysdeps/x86_64/rtld-strlen.S
+++ /dev/null
@@ -1,136 +0,0 @@
-/* strlen(str) -- determine the length of the string STR.
- Copyright (C) 2002-2015 Free Software Foundation, Inc.
- Based on i486 version contributed by Ulrich Drepper <drepper@redhat.com>.
- This file is part of the GNU C Library.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#include <sysdep.h>
-#include "asm-syntax.h"
-
-
- .text
-ENTRY (strlen)
- movq %rdi, %rcx /* Duplicate source pointer. */
- andl $7, %ecx /* mask alignment bits */
- movq %rdi, %rax /* duplicate destination. */
- jz 1f /* aligned => start loop */
-
- neg %ecx /* We need to align to 8 bytes. */
- addl $8,%ecx
- /* Search the first bytes directly. */
-0: cmpb $0x0,(%rax) /* is byte NUL? */
- je 2f /* yes => return */
- incq %rax /* increment pointer */
- decl %ecx
- jnz 0b
-
-1: movq $0xfefefefefefefeff,%r8 /* Save magic. */
-
- .p2align 4 /* Align loop. */
-4: /* Main Loop is unrolled 4 times. */
- /* First unroll. */
- movq (%rax), %rcx /* get double word (= 8 bytes) in question */
- addq $8,%rax /* adjust pointer for next word */
- movq %r8, %rdx /* magic value */
- addq %rcx, %rdx /* add the magic value to the word. We get
- carry bits reported for each byte which
- is *not* 0 */
- jnc 3f /* highest byte is NUL => return pointer */
- xorq %rcx, %rdx /* (word+magic)^word */
- orq %r8, %rdx /* set all non-carry bits */
- incq %rdx /* add 1: if one carry bit was *not* set
- the addition will not result in 0. */
- jnz 3f /* found NUL => return pointer */
-
- /* Second unroll. */
- movq (%rax), %rcx /* get double word (= 8 bytes) in question */
- addq $8,%rax /* adjust pointer for next word */
- movq %r8, %rdx /* magic value */
- addq %rcx, %rdx /* add the magic value to the word. We get
- carry bits reported for each byte which
- is *not* 0 */
- jnc 3f /* highest byte is NUL => return pointer */
- xorq %rcx, %rdx /* (word+magic)^word */
- orq %r8, %rdx /* set all non-carry bits */
- incq %rdx /* add 1: if one carry bit was *not* set
- the addition will not result in 0. */
- jnz 3f /* found NUL => return pointer */
-
- /* Third unroll. */
- movq (%rax), %rcx /* get double word (= 8 bytes) in question */
- addq $8,%rax /* adjust pointer for next word */
- movq %r8, %rdx /* magic value */
- addq %rcx, %rdx /* add the magic value to the word. We get
- carry bits reported for each byte which
- is *not* 0 */
- jnc 3f /* highest byte is NUL => return pointer */
- xorq %rcx, %rdx /* (word+magic)^word */
- orq %r8, %rdx /* set all non-carry bits */
- incq %rdx /* add 1: if one carry bit was *not* set
- the addition will not result in 0. */
- jnz 3f /* found NUL => return pointer */
-
- /* Fourth unroll. */
- movq (%rax), %rcx /* get double word (= 8 bytes) in question */
- addq $8,%rax /* adjust pointer for next word */
- movq %r8, %rdx /* magic value */
- addq %rcx, %rdx /* add the magic value to the word. We get
- carry bits reported for each byte which
- is *not* 0 */
- jnc 3f /* highest byte is NUL => return pointer */
- xorq %rcx, %rdx /* (word+magic)^word */
- orq %r8, %rdx /* set all non-carry bits */
- incq %rdx /* add 1: if one carry bit was *not* set
- the addition will not result in 0. */
- jz 4b /* no NUL found => continue loop */
-
- .p2align 4 /* Align, it's a jump target. */
-3: subq $8,%rax /* correct pointer increment. */
-
- testb %cl, %cl /* is first byte NUL? */
- jz 2f /* yes => return */
- incq %rax /* increment pointer */
-
- testb %ch, %ch /* is second byte NUL? */
- jz 2f /* yes => return */
- incq %rax /* increment pointer */
-
- testl $0x00ff0000, %ecx /* is third byte NUL? */
- jz 2f /* yes => return pointer */
- incq %rax /* increment pointer */
-
- testl $0xff000000, %ecx /* is fourth byte NUL? */
- jz 2f /* yes => return pointer */
- incq %rax /* increment pointer */
-
- shrq $32, %rcx /* look at other half. */
-
- testb %cl, %cl /* is first byte NUL? */
- jz 2f /* yes => return */
- incq %rax /* increment pointer */
-
- testb %ch, %ch /* is second byte NUL? */
- jz 2f /* yes => return */
- incq %rax /* increment pointer */
-
- testl $0xff0000, %ecx /* is third byte NUL? */
- jz 2f /* yes => return pointer */
- incq %rax /* increment pointer */
-2:
- subq %rdi, %rax /* compute difference to string start */
- ret
-END (strlen)
-libc_hidden_builtin_def (strlen)
diff --git a/sysdeps/x86_64/strcmp.S b/sysdeps/x86_64/strcmp.S
index 1329649..02bb9a6 100644
--- a/sysdeps/x86_64/strcmp.S
+++ b/sysdeps/x86_64/strcmp.S
@@ -128,6 +128,7 @@ libc_hidden_def (__strncasecmp)
ENTRY (STRCMP)
#if !IS_IN (libc)
/* Simple version since we can't use SSE registers in ld.so. */
+ jmp __strcmp_sse2_unaligned
L(oop): movb (%rdi), %al
cmpb (%rsi), %al
jne L(neq)
--
2.1.4
More information about the Libc-alpha
mailing list