commit 00e93b16bdb1ecdf8f0d663e74595988316d9d9e Author: Richard Henderson Date: Sun Aug 22 15:11:12 2010 -0700 Extract routine for variable shift. diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile index b124524..27dc563 100644 --- a/sysdeps/x86_64/multiarch/Makefile +++ b/sysdeps/x86_64/multiarch/Makefile @@ -10,7 +10,7 @@ sysdep_routines += stpncpy-c strncpy-c strcmp-ssse3 strncmp-ssse3 \ memmove-ssse3-back strcasestr-nonascii strcasecmp_l-ssse3 \ strncase_l-ssse3 ifeq (yes,$(config-cflags-sse4)) -sysdep_routines += strcspn-c strpbrk-c strspn-c strstr-c strcasestr-c +sysdep_routines += strcspn-c strpbrk-c strspn-c strstr-c strcasestr-c varshift CFLAGS-strcspn-c.c += -msse4 CFLAGS-strpbrk-c.c += -msse4 CFLAGS-strspn-c.c += -msse4 diff --git a/sysdeps/x86_64/multiarch/strcspn-c.c b/sysdeps/x86_64/multiarch/strcspn-c.c index daeebe1..04aba46 100644 --- a/sysdeps/x86_64/multiarch/strcspn-c.c +++ b/sysdeps/x86_64/multiarch/strcspn-c.c @@ -20,6 +20,7 @@ #include #include +#include "varshift.h" /* We use 0x2: _SIDD_SBYTE_OPS @@ -86,8 +87,6 @@ STRCSPN_SSE42 (const char *s, const char *a) const char *aligned; __m128i mask; - /* Fake initialization. gcc otherwise will warn. */ - asm ("" : "=xm" (mask)); int offset = (int) ((size_t) a & 15); if (offset != 0) { @@ -95,54 +94,7 @@ STRCSPN_SSE42 (const char *s, const char *a) aligned = (const char *) ((size_t) a & -16L); __m128i mask0 = _mm_load_si128 ((__m128i *) aligned); - switch (offset) - { - case 1: - mask = _mm_srli_si128 (mask0, 1); - break; - case 2: - mask = _mm_srli_si128 (mask0, 2); - break; - case 3: - mask = _mm_srli_si128 (mask0, 3); - break; - case 4: - mask = _mm_srli_si128 (mask0, 4); - break; - case 5: - mask = _mm_srli_si128 (mask0, 5); - break; - case 6: - mask = _mm_srli_si128 (mask0, 6); - break; - case 7: - mask = _mm_srli_si128 (mask0, 7); - break; - case 8: - mask = _mm_srli_si128 (mask0, 8); - break; - case 9: - mask = _mm_srli_si128 (mask0, 9); - break; - case 10: - mask = _mm_srli_si128 (mask0, 10); - break; - case 11: - mask = _mm_srli_si128 (mask0, 11); - break; - case 12: - mask = _mm_srli_si128 (mask0, 12); - break; - case 13: - mask = _mm_srli_si128 (mask0, 13); - break; - case 14: - mask = _mm_srli_si128 (mask0, 14); - break; - case 15: - mask = _mm_srli_si128 (mask0, 15); - break; - } + mask = __m128i_shift_right (mask0, offset); /* Find where the NULL terminator is. */ int length = _mm_cmpistri (mask, mask, 0x3a); @@ -159,55 +111,10 @@ STRCSPN_SSE42 (const char *s, const char *a) if (index != 0) { - /* Combine mask0 and mask1. */ - switch (offset) - { - case 1: - mask = _mm_alignr_epi8 (mask1, mask0, 1); - break; - case 2: - mask = _mm_alignr_epi8 (mask1, mask0, 2); - break; - case 3: - mask = _mm_alignr_epi8 (mask1, mask0, 3); - break; - case 4: - mask = _mm_alignr_epi8 (mask1, mask0, 4); - break; - case 5: - mask = _mm_alignr_epi8 (mask1, mask0, 5); - break; - case 6: - mask = _mm_alignr_epi8 (mask1, mask0, 6); - break; - case 7: - mask = _mm_alignr_epi8 (mask1, mask0, 7); - break; - case 8: - mask = _mm_alignr_epi8 (mask1, mask0, 8); - break; - case 9: - mask = _mm_alignr_epi8 (mask1, mask0, 9); - break; - case 10: - mask = _mm_alignr_epi8 (mask1, mask0, 10); - break; - case 11: - mask = _mm_alignr_epi8 (mask1, mask0, 11); - break; - case 12: - mask = _mm_alignr_epi8 (mask1, mask0, 12); - break; - case 13: - mask = _mm_alignr_epi8 (mask1, mask0, 13); - break; - case 14: - mask = _mm_alignr_epi8 (mask1, mask0, 14); - break; - case 15: - mask = _mm_alignr_epi8 (mask1, mask0, 15); - break; - } + /* Combine mask0 and mask1. We could play games with + palignr, but frankly this data should be in L1 now + so do the merge via an unaligned load. */ + mask = _mm_loadu_si128 ((__m128i *) a); } } } @@ -234,54 +141,7 @@ STRCSPN_SSE42 (const char *s, const char *a) aligned = (const char *) ((size_t) s & -16L); __m128i value = _mm_load_si128 ((__m128i *) aligned); - switch (offset) - { - case 1: - value = _mm_srli_si128 (value, 1); - break; - case 2: - value = _mm_srli_si128 (value, 2); - break; - case 3: - value = _mm_srli_si128 (value, 3); - break; - case 4: - value = _mm_srli_si128 (value, 4); - break; - case 5: - value = _mm_srli_si128 (value, 5); - break; - case 6: - value = _mm_srli_si128 (value, 6); - break; - case 7: - value = _mm_srli_si128 (value, 7); - break; - case 8: - value = _mm_srli_si128 (value, 8); - break; - case 9: - value = _mm_srli_si128 (value, 9); - break; - case 10: - value = _mm_srli_si128 (value, 10); - break; - case 11: - value = _mm_srli_si128 (value, 11); - break; - case 12: - value = _mm_srli_si128 (value, 12); - break; - case 13: - value = _mm_srli_si128 (value, 13); - break; - case 14: - value = _mm_srli_si128 (value, 14); - break; - case 15: - value = _mm_srli_si128 (value, 15); - break; - } + value = __m128i_shift_right (value, offset); int length = _mm_cmpistri (mask, value, 0x2); /* No need to check ZFlag since ZFlag is always 1. */ diff --git a/sysdeps/x86_64/multiarch/strspn-c.c b/sysdeps/x86_64/multiarch/strspn-c.c index be9e8ac..ab58549 100644 --- a/sysdeps/x86_64/multiarch/strspn-c.c +++ b/sysdeps/x86_64/multiarch/strspn-c.c @@ -20,6 +20,7 @@ #include #include +#include "varshift.h" /* We use 0x12: _SIDD_SBYTE_OPS @@ -71,54 +72,7 @@ __strspn_sse42 (const char *s, const char *a) aligned = (const char *) ((size_t) a & -16L); __m128i mask0 = _mm_load_si128 ((__m128i *) aligned); - switch (offset) - { - case 1: - mask = _mm_srli_si128 (mask0, 1); - break; - case 2: - mask = _mm_srli_si128 (mask0, 2); - break; - case 3: - mask = _mm_srli_si128 (mask0, 3); - break; - case 4: - mask = _mm_srli_si128 (mask0, 4); - break; - case 5: - mask = _mm_srli_si128 (mask0, 5); - break; - case 6: - mask = _mm_srli_si128 (mask0, 6); - break; - case 7: - mask = _mm_srli_si128 (mask0, 7); - break; - case 8: - mask = _mm_srli_si128 (mask0, 8); - break; - case 9: - mask = _mm_srli_si128 (mask0, 9); - break; - case 10: - mask = _mm_srli_si128 (mask0, 10); - break; - case 11: - mask = _mm_srli_si128 (mask0, 11); - break; - case 12: - mask = _mm_srli_si128 (mask0, 12); - break; - case 13: - mask = _mm_srli_si128 (mask0, 13); - break; - case 14: - mask = _mm_srli_si128 (mask0, 14); - break; - case 15: - mask = _mm_srli_si128 (mask0, 15); - break; - } + mask = __m128i_shift_right (mask0, offset); /* Find where the NULL terminator is. */ int length = _mm_cmpistri (mask, mask, 0x3a); @@ -135,55 +89,10 @@ __strspn_sse42 (const char *s, const char *a) if (index != 0) { - /* Combine mask0 and mask1. */ - switch (offset) - { - case 1: - mask = _mm_alignr_epi8 (mask1, mask0, 1); - break; - case 2: - mask = _mm_alignr_epi8 (mask1, mask0, 2); - break; - case 3: - mask = _mm_alignr_epi8 (mask1, mask0, 3); - break; - case 4: - mask = _mm_alignr_epi8 (mask1, mask0, 4); - break; - case 5: - mask = _mm_alignr_epi8 (mask1, mask0, 5); - break; - case 6: - mask = _mm_alignr_epi8 (mask1, mask0, 6); - break; - case 7: - mask = _mm_alignr_epi8 (mask1, mask0, 7); - break; - case 8: - mask = _mm_alignr_epi8 (mask1, mask0, 8); - break; - case 9: - mask = _mm_alignr_epi8 (mask1, mask0, 9); - break; - case 10: - mask = _mm_alignr_epi8 (mask1, mask0, 10); - break; - case 11: - mask = _mm_alignr_epi8 (mask1, mask0, 11); - break; - case 12: - mask = _mm_alignr_epi8 (mask1, mask0, 12); - break; - case 13: - mask = _mm_alignr_epi8 (mask1, mask0, 13); - break; - case 14: - mask = _mm_alignr_epi8 (mask1, mask0, 14); - break; - case 15: - mask = _mm_alignr_epi8 (mask1, mask0, 15); - break; - } + /* Combine mask0 and mask1. We could play games with + palignr, but frankly this data should be in L1 now + so do the merge via an unaligned load. */ + mask = _mm_loadu_si128 ((__m128i *) a); } } } @@ -210,54 +119,7 @@ __strspn_sse42 (const char *s, const char *a) aligned = (const char *) ((size_t) s & -16L); __m128i value = _mm_load_si128 ((__m128i *) aligned); - switch (offset) - { - case 1: - value = _mm_srli_si128 (value, 1); - break; - case 2: - value = _mm_srli_si128 (value, 2); - break; - case 3: - value = _mm_srli_si128 (value, 3); - break; - case 4: - value = _mm_srli_si128 (value, 4); - break; - case 5: - value = _mm_srli_si128 (value, 5); - break; - case 6: - value = _mm_srli_si128 (value, 6); - break; - case 7: - value = _mm_srli_si128 (value, 7); - break; - case 8: - value = _mm_srli_si128 (value, 8); - break; - case 9: - value = _mm_srli_si128 (value, 9); - break; - case 10: - value = _mm_srli_si128 (value, 10); - break; - case 11: - value = _mm_srli_si128 (value, 11); - break; - case 12: - value = _mm_srli_si128 (value, 12); - break; - case 13: - value = _mm_srli_si128 (value, 13); - break; - case 14: - value = _mm_srli_si128 (value, 14); - break; - case 15: - value = _mm_srli_si128 (value, 15); - break; - } + value = __m128i_shift_right (value, offset); int length = _mm_cmpistri (mask, value, 0x12); /* No need to check CFlag since it is always 1. */ diff --git a/sysdeps/x86_64/multiarch/strstr.c b/sysdeps/x86_64/multiarch/strstr.c index 45d7a55..b408b75 100644 --- a/sysdeps/x86_64/multiarch/strstr.c +++ b/sysdeps/x86_64/multiarch/strstr.c @@ -19,6 +19,7 @@ 02111-1307 USA. */ #include +#include "varshift.h" #ifndef STRSTR_SSE42 # define STRSTR_SSE42 __strstr_sse42 @@ -82,67 +83,6 @@ 5. failed string compare, go back to scanning */ -/* Fix-up of removal of unneeded data due to 16B aligned load - parameters: - value: 16B data loaded from 16B aligned address. - offset: Offset of target data address relative to 16B aligned load - address. - */ - -static __inline__ __m128i -__m128i_shift_right (__m128i value, int offset) -{ - switch (offset) - { - case 1: - value = _mm_srli_si128 (value, 1); - break; - case 2: - value = _mm_srli_si128 (value, 2); - break; - case 3: - value = _mm_srli_si128 (value, 3); - break; - case 4: - value = _mm_srli_si128 (value, 4); - break; - case 5: - value = _mm_srli_si128 (value, 5); - break; - case 6: - value = _mm_srli_si128 (value, 6); - break; - case 7: - value = _mm_srli_si128 (value, 7); - break; - case 8: - value = _mm_srli_si128 (value, 8); - break; - case 9: - value = _mm_srli_si128 (value, 9); - break; - case 10: - value = _mm_srli_si128 (value, 10); - break; - case 11: - value = _mm_srli_si128 (value, 11); - break; - case 12: - value = _mm_srli_si128 (value, 12); - break; - case 13: - value = _mm_srli_si128 (value, 13); - break; - case 14: - value = _mm_srli_si128 (value, 14); - break; - case 15: - value = _mm_srli_si128 (value, 15); - break; - } - return value; -} - /* Simple replacement of movdqu to address 4KB boundary cross issue. If EOS occurs within less than 16B before 4KB boundary, we don't cross to next page. */ diff --git a/sysdeps/x86_64/multiarch/varshift.S b/sysdeps/x86_64/multiarch/varshift.S new file mode 100644 index 0000000..09a7de9 --- /dev/null +++ b/sysdeps/x86_64/multiarch/varshift.S @@ -0,0 +1,78 @@ +/* Helper for variable shifts of SSE registers. + Copyright (C) 2010 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include + + + .text +/* Each table entry is 8 bytes. */ + .balign 8 + .hidden ___m128i_shift_right + +ENTRY(___m128i_shift_right) + + rep; ret + + .balign 8 + psrldq $1, %xmm0 + ret + .balign 8 + psrldq $2, %xmm0 + ret + .balign 8 + psrldq $3, %xmm0 + ret + .balign 8 + psrldq $4, %xmm0 + ret + .balign 8 + psrldq $5, %xmm0 + ret + .balign 8 + psrldq $6, %xmm0 + ret + .balign 8 + psrldq $7, %xmm0 + ret + .balign 8 + psrldq $8, %xmm0 + ret + .balign 8 + psrldq $9, %xmm0 + ret + .balign 8 + psrldq $10, %xmm0 + ret + .balign 8 + psrldq $11, %xmm0 + ret + .balign 8 + psrldq $12, %xmm0 + ret + .balign 8 + psrldq $13, %xmm0 + ret + .balign 8 + psrldq $14, %xmm0 + ret + .balign 8 + psrldq $15, %xmm0 + ret + +END(___m128i_shift_right) diff --git a/sysdeps/x86_64/multiarch/varshift.h b/sysdeps/x86_64/multiarch/varshift.h new file mode 100644 index 0000000..7ce5b76 --- /dev/null +++ b/sysdeps/x86_64/multiarch/varshift.h @@ -0,0 +1,34 @@ +/* Helper for variable shifts of SSE registers. + Copyright (C) 2010 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + + +static __inline__ __m128i +__m128i_shift_right (__m128i value, unsigned long offset) +{ + register __m128i xmm0 __asm__("xmm0") = value; + unsigned long tmp; + + /* Each table entry is 8 bytes. */ + __asm ( "lea ___m128i_shift_right(%%rip), %0\n" + " lea (%0, %2, 8), %0\n" + " call *%0" + : "=&r"(tmp), "+x"(xmm0) : "r"(offset)); + + return xmm0; +}