This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
Re: PATCH: Improve x86-64 bcopy/bzero
On Wed, Oct 10, 2012 at 3:55 PM, Roland McGrath <roland@hack.frob.com> wrote:
>> --- /dev/null
>> +++ b/string/test-bcopy.c
>> @@ -0,0 +1,262 @@
>> +/* Test and measure bcopy functions.
>
> Please modify test-memmove.c with some macro use so this can
> just #define some things and #include it instead of duplicating it.
> Likewise for test-bzero.c vs test-memset.c.
>
>> ENTRY(bcopy)
>> xchg %rdi, %rsi
>> - jmp HIDDEN_BUILTIN_JUMPTARGET(memmove)
>> + jmp __libc_memmove
>
> Add a comment saying we are intentionally using the PLT entry here.
>
>> + jmp __libc_memset
>> +END(__bzero)
>
> Likewise.
>
>> --- a/sysdeps/x86_64/multiarch/memset.S
>> +++ b/sysdeps/x86_64/multiarch/memset.S
>> @@ -1,5 +1,5 @@
>> /* Multiple versions of memset
>> - Copyright (C) 2010 Free Software Foundation, Inc.
>> + Copyright (C) 2010-12 Free Software Foundation, Inc.
>
> No two-digit years.
>
Here is the updated patch. OK to install?
Thanks.
--
H.J.
---
* string/Makefile (strop-tests): Add bcopy and bzero.
* string/test-bcopy.c: New file.
* string/test-bzero.c: Likewise.
* string/test-memmove.c: Support bcopy test if TEST_BCOPY is
defined.
* string/test-memset.c: Support bzero test if TEST_BZERO is
defined.
* sysdeps/x86_64/multiarch/bcopy.S (bcopy): Jump to
__libc_memmove.
* sysdeps/x86_64/multiarch/bzero.S (__bzero): Jump to
__libc_memset.
* sysdeps/x86_64/multiarch/memset.S (__libc_memset): New alias
of memset.
diff --git a/string/Makefile b/string/Makefile
index 6f2c6c3..8d1db74 100644
--- a/string/Makefile
+++ b/string/Makefile
@@ -49,7 +49,7 @@ strop-tests := memchr memcmp memcpy memmove mempcpy
memset memccpy \
stpcpy stpncpy strcat strchr strcmp strcpy strcspn \
strlen strncmp strncpy strpbrk strrchr strspn memmem \
strstr strcasestr strnlen strcasecmp strncasecmp \
- strncat rawmemchr strchrnul
+ strncat rawmemchr strchrnul bcopy bzero
tests := tester inl-tester noinl-tester testcopy test-ffs \
tst-strlen stratcliff tst-svc tst-inlcall \
bug-strncat1 bug-strspn1 bug-strpbrk1 tst-bswap \
diff --git a/string/test-bcopy.c b/string/test-bcopy.c
new file mode 100644
index 0000000..f609736
--- /dev/null
+++ b/string/test-bcopy.c
@@ -0,0 +1,44 @@
+/* Test and measure bcopy functions.
+ Copyright (C) 2012 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#define TEST_BCOPY
+#define TEST_MAIN
+#include "test-string.h"
+
+typedef void (*proto_t) (const char *, char *, size_t);
+void simple_bcopy (const char *, char *, size_t);
+
+IMPL (simple_bcopy, 0)
+IMPL (bcopy, 1)
+
+void
+simple_bcopy (const char *src, char *dst, size_t n)
+{
+ if (src < dst)
+ {
+ dst += n;
+ src += n;
+ while (n--)
+ *--dst = *--src;
+ }
+ else
+ while (n--)
+ *dst++ = *src++;
+}
+
+#include "test-memmove.c"
diff --git a/string/test-bzero.c b/string/test-bzero.c
new file mode 100644
index 0000000..4ffa730
--- /dev/null
+++ b/string/test-bzero.c
@@ -0,0 +1,45 @@
+/* Test and measure bzero functions.
+ Copyright (C) 2012 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+#define TEST_BZERO
+#define TEST_MAIN
+#define MIN_PAGE_SIZE 131072
+#include "test-string.h"
+
+typedef void (*proto_t) (char *, size_t);
+void simple_bzero (char *, size_t);
+void builtin_bzero (char *, size_t);
+
+IMPL (simple_bzero, 0)
+IMPL (builtin_bzero, 0)
+IMPL (bzero, 1)
+
+void
+simple_bzero (char *s, size_t n)
+{
+ char *r = s, *end = s + n;
+ while (r < end)
+ *r++ = 0;
+}
+
+void
+builtin_bzero (char *s, size_t n)
+{
+ __builtin_bzero (s, n);
+}
+
+#include "test-memset.c"
diff --git a/string/test-memmove.c b/string/test-memmove.c
index 7d10712..db98b7e 100644
--- a/string/test-memmove.c
+++ b/string/test-memmove.c
@@ -1,5 +1,5 @@
/* Test and measure memmove functions.
- Copyright (C) 1999, 2002, 2003 Free Software Foundation, Inc.
+ Copyright (C) 1999-2012 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Written by Jakub Jelinek <jakub@redhat.com>, 1999.
@@ -17,8 +17,9 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#define TEST_MAIN
-#include "test-string.h"
+#ifndef TEST_BCOPY
+# define TEST_MAIN
+# include "test-string.h"
typedef char *(*proto_t) (char *, const char *, size_t);
char *simple_memmove (char *, const char *, size_t);
@@ -42,14 +43,18 @@ simple_memmove (char *dst, const char *src, size_t n)
*dst++ = *src++;
return ret;
}
+#endif
static void
do_one_test (impl_t *impl, char *dst, char *src, const char *orig_src,
size_t len)
{
+ memcpy (src, orig_src, len);
+#ifdef TEST_BCOPY
+ CALL (impl, src, dst, len);
+#else
char *res;
- memcpy (src, orig_src, len);
res = CALL (impl, dst, src, len);
if (res != dst)
{
@@ -58,6 +63,7 @@ do_one_test (impl_t *impl, char *dst, char *src,
const char *orig_src,
ret = 1;
return;
}
+#endif
if (memcmp (dst, orig_src, len) != 0)
{
@@ -77,7 +83,11 @@ do_one_test (impl_t *impl, char *dst, char *src,
const char *orig_src,
for (i = 0; i < 32; ++i)
{
HP_TIMING_NOW (start);
+#ifdef TEST_BCOPY
+ CALL (impl, src, dst, len);
+#else
CALL (impl, dst, src, len);
+#endif
HP_TIMING_NOW (stop);
HP_TIMING_BEST (best_time, start, stop);
}
@@ -123,7 +133,9 @@ do_random_tests (void)
size_t srcstart, srcend, dststart, dstend;
int c;
unsigned char *p1, *p2;
+#ifndef TEST_BCOPY
unsigned char *res;
+#endif
for (n = 0; n < ITERATIONS; n++)
{
@@ -178,6 +190,9 @@ do_random_tests (void)
{
memset (p2 + dststart, c, dstend - dststart);
memcpy (p2 + srcstart, p1 + srcstart, srcend - srcstart);
+#ifdef TEST_BCOPY
+ CALL (impl, (char *) (p2 + align1), (char *) (p2 + align2), len);
+#else
res = (unsigned char *) CALL (impl,
(char *) (p2 + align2),
(char *) (p2 + align1), len);
@@ -187,6 +202,7 @@ do_random_tests (void)
n, impl->name, align1, align2, len, res, p2 + align2);
ret = 1;
}
+#endif
if (memcmp (p1 + align1, p2 + align2, len))
{
error (0, 0, "Iteration %zd - different strings, %s (%zd, %zd, %zd)",
diff --git a/string/test-memset.c b/string/test-memset.c
index 839b8a1..d7394e1 100644
--- a/string/test-memset.c
+++ b/string/test-memset.c
@@ -1,5 +1,5 @@
/* Test and measure memset functions.
- Copyright (C) 1999, 2002, 2003, 2005 Free Software Foundation, Inc.
+ Copyright (C) 1999-2012 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Written by Jakub Jelinek <jakub@redhat.com>, 1999.
@@ -17,9 +17,10 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
-#define TEST_MAIN
-#define MIN_PAGE_SIZE 131072
-#include "test-string.h"
+#ifndef TEST_BZERO
+# define TEST_MAIN
+# define MIN_PAGE_SIZE 131072
+# include "test-string.h"
typedef char *(*proto_t) (char *, int, size_t);
char *simple_memset (char *, int, size_t);
@@ -43,15 +44,22 @@ builtin_memset (char *s, int c, size_t n)
{
return __builtin_memset (s, c, n);
}
+#endif
static void
-do_one_test (impl_t *impl, char *s, int c, size_t n)
+do_one_test (impl_t *impl, char *s, int c __attribute ((unused)), size_t n)
{
- char *res = CALL (impl, s, c, n);
char tstbuf[n];
+#ifdef TEST_BZERO
+ simple_bzero (tstbuf, n);
+ CALL (impl, s, n);
+ if (memcmp (s, tstbuf, n) != 0)
+#else
+ char *res = CALL (impl, s, c, n);
if (res != s
|| simple_memset (tstbuf, c, n) != tstbuf
|| memcmp (s, tstbuf, n) != 0)
+#endif
{
error (0, 0, "Wrong result in function %s", impl->name);
ret = 1;
@@ -68,7 +76,12 @@ do_one_test (impl_t *impl, char *s, int c, size_t n)
for (i = 0; i < 32; ++i)
{
HP_TIMING_NOW (start);
+#ifdef TEST_BZERO
+ CALL (impl, s, n);
+#else
CALL (impl, s, c, n);
+#endif
+
HP_TIMING_NOW (stop);
HP_TIMING_BEST (best_time, start, stop);
}
@@ -94,6 +107,7 @@ do_test (size_t align, int c, size_t len)
putchar ('\n');
}
+#ifndef TEST_BZERO
static void
do_random_tests (void)
{
@@ -178,12 +192,13 @@ do_random_tests (void)
}
}
}
+#endif
int
test_main (void)
{
size_t i;
- int c;
+ int c = 0;
test_init ();
@@ -192,7 +207,9 @@ test_main (void)
printf ("\t%s", impl->name);
putchar ('\n');
+#ifndef TEST_BZERO
for (c = -65; c <= 130; c += 65)
+#endif
{
for (i = 0; i < 18; ++i)
do_test (0, c, 1 << i);
@@ -208,7 +225,10 @@ test_main (void)
do_test (2, c, 25);
}
+#ifndef TEST_BZERO
do_random_tests ();
+#endif
+
return ret;
}
diff --git a/sysdeps/x86_64/multiarch/bcopy.S b/sysdeps/x86_64/multiarch/bcopy.S
index 11e250f..639f02b 100644
--- a/sysdeps/x86_64/multiarch/bcopy.S
+++ b/sysdeps/x86_64/multiarch/bcopy.S
@@ -3,5 +3,5 @@
.text
ENTRY(bcopy)
xchg %rdi, %rsi
- jmp HIDDEN_BUILTIN_JUMPTARGET(memmove)
+ jmp __libc_memmove /* Branch to IFUNC memmove. */
END(bcopy)
diff --git a/sysdeps/x86_64/multiarch/bzero.S b/sysdeps/x86_64/multiarch/bzero.S
index 92e9fcf..23beab7 100644
--- a/sysdeps/x86_64/multiarch/bzero.S
+++ b/sysdeps/x86_64/multiarch/bzero.S
@@ -1,5 +1,5 @@
-/* Multiple versions of bzero
- Copyright (C) 2010 Free Software Foundation, Inc.
+/* bzero. x86-64 version.
+ Copyright (C) 2010-2012 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@@ -21,35 +21,8 @@
.text
ENTRY(__bzero)
- .type __bzero, @gnu_indirect_function
- cmpl $0, __cpu_features+KIND_OFFSET(%rip)
- jne 1f
- call __init_cpu_features
-1: leaq __bzero_x86_64(%rip), %rax
- testl $bit_Prefer_SSE_for_memop,
__cpu_features+FEATURE_OFFSET+index_Prefer_SSE_for_memop(%rip)
- jz 2f
- leaq __bzero_sse2(%rip), %rax
-2: ret
-END(__bzero)
-
- .type __bzero_sse2, @function
-__bzero_sse2:
- cfi_startproc
- CALL_MCOUNT
mov %rsi,%rdx /* Adjust parameter. */
xorl %esi,%esi /* Fill with 0s. */
- jmp __memset_sse2
- cfi_endproc
- .size __bzero_sse2, .-__bzero_sse2
-
- .type __bzero_x86_64, @function
-__bzero_x86_64:
- cfi_startproc
- CALL_MCOUNT
- mov %rsi,%rdx /* Adjust parameter. */
- xorl %esi,%esi /* Fill with 0s. */
- jmp __memset_x86_64
- cfi_endproc
- .size __bzero_x86_64, .-__bzero_x86_64
-
+ jmp __libc_memset /* Branch to IFUNC memset. */
+END(__bzero)
weak_alias (__bzero, bzero)
diff --git a/sysdeps/x86_64/multiarch/memset.S
b/sysdeps/x86_64/multiarch/memset.S
index 1a7fa2d..10f00bf 100644
--- a/sysdeps/x86_64/multiarch/memset.S
+++ b/sysdeps/x86_64/multiarch/memset.S
@@ -1,5 +1,5 @@
/* Multiple versions of memset
- Copyright (C) 2010 Free Software Foundation, Inc.
+ Copyright (C) 2010-2012 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
@@ -33,6 +33,11 @@ ENTRY(memset)
2: ret
END(memset)
+/* Define internal IFUNC memset for bzero. */
+ .globl __libc_memset
+ .hidden __libc_memset
+ __libc_memset = memset
+
# define USE_SSE2 1
# undef ENTRY